Files
Market-Data-Downloader/ui_template_gaps.py
2025-10-05 13:10:12 +01:00

781 lines
27 KiB
Python

#!/usr/bin/env python3
"""
ui_template_gaps.py - Data Gap Monitoring Interface
Provides visual interface for tracking and filling data gaps
"""
def get_gaps_monitoring_html():
"""Return the gaps monitoring page HTML"""
return """
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Gap Monitoring - Trading System</title>
<style>
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
padding: 20px;
}
.container {
max-width: 1400px;
margin: 0 auto;
background: white;
border-radius: 10px;
box-shadow: 0 10px 40px rgba(0,0,0,0.1);
overflow: hidden;
}
.header {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
padding: 30px;
text-align: center;
}
.header h1 {
font-size: 32px;
margin-bottom: 10px;
}
.header p {
opacity: 0.9;
font-size: 16px;
}
.nav-bar {
background: #2d3748;
padding: 15px 30px;
display: flex;
gap: 10px;
flex-wrap: wrap;
}
.nav-button {
background: #4a5568;
color: white;
border: none;
padding: 10px 20px;
border-radius: 5px;
cursor: pointer;
font-size: 14px;
transition: all 0.3s;
}
.nav-button:hover {
background: #667eea;
transform: translateY(-2px);
}
.nav-button.active {
background: #667eea;
}
.stats-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 20px;
padding: 30px;
background: #f7fafc;
}
.stat-card {
background: white;
padding: 20px;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
text-align: center;
}
.stat-value {
font-size: 32px;
font-weight: bold;
color: #667eea;
margin: 10px 0;
}
.stat-label {
color: #718096;
font-size: 14px;
}
.content {
padding: 30px;
}
.section {
margin-bottom: 30px;
}
.section-title {
font-size: 20px;
font-weight: bold;
color: #2d3748;
margin-bottom: 15px;
padding-bottom: 10px;
border-bottom: 2px solid #e2e8f0;
}
.pairs-table {
width: 100%;
border-collapse: collapse;
background: white;
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
border-radius: 8px;
overflow: hidden;
}
.pairs-table thead {
background: #2d3748;
color: white;
}
.pairs-table th {
padding: 15px;
text-align: left;
font-weight: 600;
}
.pairs-table td {
padding: 12px 15px;
border-bottom: 1px solid #e2e8f0;
}
.pairs-table tbody tr:hover {
background: #f7fafc;
}
.status-badge {
display: inline-block;
padding: 4px 12px;
border-radius: 12px;
font-size: 12px;
font-weight: 600;
}
.status-excellent {
background: #c6f6d5;
color: #22543d;
}
.status-good {
background: #bee3f8;
color: #2c5282;
}
.status-partial {
background: #feebc8;
color: #744210;
}
.status-poor {
background: #fed7d7;
color: #742a2a;
}
.action-button {
background: #667eea;
color: white;
border: none;
padding: 6px 12px;
border-radius: 4px;
cursor: pointer;
font-size: 12px;
margin-right: 5px;
transition: all 0.3s;
}
.action-button:hover {
background: #5a67d8;
transform: translateY(-1px);
}
.action-button.secondary {
background: #48bb78;
}
.action-button.secondary:hover {
background: #38a169;
}
.loading {
text-align: center;
padding: 40px;
color: #718096;
}
.spinner {
border: 4px solid #e2e8f0;
border-top: 4px solid #667eea;
border-radius: 50%;
width: 40px;
height: 40px;
animation: spin 1s linear infinite;
margin: 0 auto 20px;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
.modal {
display: none;
position: fixed;
z-index: 1000;
left: 0;
top: 0;
width: 100%;
height: 100%;
background: rgba(0,0,0,0.5);
}
.modal-content {
background: white;
margin: 5% auto;
padding: 30px;
border-radius: 10px;
max-width: 800px;
max-height: 80vh;
overflow-y: auto;
box-shadow: 0 10px 40px rgba(0,0,0,0.2);
}
.modal-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 20px;
padding-bottom: 15px;
border-bottom: 2px solid #e2e8f0;
}
.modal-title {
font-size: 24px;
font-weight: bold;
color: #2d3748;
}
.close {
font-size: 28px;
font-weight: bold;
color: #718096;
cursor: pointer;
transition: color 0.3s;
}
.close:hover {
color: #2d3748;
}
.gap-details {
margin-top: 20px;
}
.gap-item {
background: #f7fafc;
padding: 15px;
border-radius: 8px;
margin-bottom: 10px;
border-left: 4px solid #667eea;
}
.gap-info {
display: grid;
grid-template-columns: repeat(2, 1fr);
gap: 10px;
margin-top: 10px;
}
.gap-info-item {
display: flex;
justify-content: space-between;
}
.gap-label {
color: #718096;
font-weight: 600;
}
.gap-value {
color: #2d3748;
}
.progress-bar {
width: 100%;
height: 8px;
background: #e2e8f0;
border-radius: 4px;
overflow: hidden;
margin: 5px 0;
}
.progress-fill {
height: 100%;
background: linear-gradient(90deg, #667eea, #48bb78);
transition: width 0.3s;
}
.interval-badge {
display: inline-block;
padding: 2px 8px;
background: #edf2f7;
border-radius: 4px;
font-size: 11px;
font-weight: 600;
color: #2d3748;
margin-right: 5px;
}
</style>
</head>
<body>
<div class="container">
<div class="header">
<h1>📊 Data Gap Monitoring</h1>
<p>Track and fill data gaps across all trading pairs</p>
</div>
<div class="nav-bar">
<button onclick="window.location.href='/'" class="nav-button">Dashboard</button>
<button onclick="window.location.href='/config'" class="nav-button">Config</button>
<button onclick="window.location.href='/gaps'" class="nav-button active">Gaps</button>
</div>
<div class="stats-grid">
<div class="stat-card">
<div class="stat-label">Total Pairs</div>
<div class="stat-value" id="totalPairs">-</div>
</div>
<div class="stat-card">
<div class="stat-label">Pairs with Gaps</div>
<div class="stat-value" id="pairsWithGaps">-</div>
</div>
<div class="stat-card">
<div class="stat-label">Total Missing Records</div>
<div class="stat-value" id="totalMissing">-</div>
</div>
<div class="stat-card">
<div class="stat-label">Avg Coverage</div>
<div class="stat-value" id="avgCoverage">-</div>
</div>
</div>
<div class="content">
<div class="section">
<div class="section-title">Trading Pairs Gap Status</div>
<div id="loadingIndicator" class="loading">
<div class="spinner"></div>
<p>Loading gap data...</p>
</div>
<table class="pairs-table" id="pairsTable" style="display:none;">
<thead>
<tr>
<th>Symbol</th>
<th>From Date</th>
<th>To Date</th>
<th>Intervals</th>
<th>Coverage</th>
<th>Status</th>
<th>Actions</th>
</tr>
</thead>
<tbody id="pairsTableBody">
</tbody>
</table>
</div>
</div>
</div>
<!-- Gap Details Modal -->
<div id="gapModal" class="modal">
<div class="modal-content">
<div class="modal-header">
<div class="modal-title" id="modalTitle">Gap Details</div>
<span class="close" onclick="closeModal()">&times;</span>
</div>
<div id="modalBody">
<div class="loading">
<div class="spinner"></div>
<p>Loading details...</p>
</div>
</div>
</div>
</div>
<script>
let gapData = [];
async function loadGapData() {
try {
const response = await fetch('/api/gaps/all-pairs');
const result = await response.json();
if (result.status === 'success' && result.data) {
gapData = result.data;
processAndDisplayData(gapData);
} else {
showError('Failed to load gap data: ' + (result.message || 'Unknown error'));
}
} catch (error) {
console.error('Error loading gap data:', error);
showError('Error loading gap data: ' + error.message);
}
}
function processAndDisplayData(data) {
// Group by symbol
const symbolMap = new Map();
data.forEach(item => {
if (!symbolMap.has(item.symbol)) {
symbolMap.set(item.symbol, {
symbol: item.symbol,
intervals: [],
totalCoverage: 0,
totalMissing: 0,
totalRecords: 0,
firstDate: item.first_record,
lastDate: item.last_record,
healthScore: null
});
}
const symbolData = symbolMap.get(item.symbol);
symbolData.intervals.push({
interval: item.interval,
coverage: item.coverage_percent || 0,
missing: item.missing_records || 0,
total: item.total_records || 0,
gaps: item.gaps || []
});
symbolData.totalMissing += item.missing_records || 0;
symbolData.totalRecords += item.total_records || 0;
// Update date ranges
if (!symbolData.firstDate || (item.first_record && item.first_record < symbolData.firstDate)) {
symbolData.firstDate = item.first_record;
}
if (!symbolData.lastDate || (item.last_record && item.last_record > symbolData.lastDate)) {
symbolData.lastDate = item.last_record;
}
});
// Calculate average coverage per symbol and load health scores
symbolMap.forEach(async (symbolData) => {
if (symbolData.intervals.length > 0) {
const totalCoverage = symbolData.intervals.reduce((sum, i) => sum + i.coverage, 0);
symbolData.totalCoverage = totalCoverage / symbolData.intervals.length;
}
// Load health score for first interval (as a sample)
if (symbolData.intervals[0]) {
await loadHealthScore(symbolData.symbol, symbolData.intervals[0].interval, symbolData);
}
});
// Update statistics
updateStatistics(symbolMap);
// Display table
displayPairsTable(symbolMap);
// Hide loading indicator
document.getElementById('loadingIndicator').style.display = 'none';
document.getElementById('pairsTable').style.display = 'table';
}
async function loadHealthScore(symbol, interval, symbolData) {
try {
const response = await fetch(`/api/gaps/health/${symbol}/${interval}`);
const result = await response.json();
if (result.status === 'success' && result.data) {
symbolData.healthScore = result.data.health_score;
symbolData.healthIssues = result.data.issues || [];
}
} catch (error) {
console.error(`Error loading health for ${symbol}:`, error);
}
}
function updateStatistics(symbolMap) {
const totalPairs = symbolMap.size;
const pairsWithGaps = Array.from(symbolMap.values()).filter(s => s.totalMissing > 0).length;
const totalMissing = Array.from(symbolMap.values()).reduce((sum, s) => sum + s.totalMissing, 0);
const avgCoverage = totalPairs > 0
? Array.from(symbolMap.values()).reduce((sum, s) => sum + s.totalCoverage, 0) / totalPairs
: 0;
document.getElementById('totalPairs').textContent = totalPairs;
document.getElementById('pairsWithGaps').textContent = pairsWithGaps;
document.getElementById('totalMissing').textContent = totalMissing.toLocaleString();
document.getElementById('avgCoverage').textContent = avgCoverage.toFixed(1) + '%';
}
function displayPairsTable(symbolMap) {
const tbody = document.getElementById('pairsTableBody');
tbody.innerHTML = '';
// Sort by coverage (worst first)
const sortedSymbols = Array.from(symbolMap.values()).sort((a, b) => a.totalCoverage - b.totalCoverage);
sortedSymbols.forEach(symbolData => {
const row = document.createElement('tr');
const coverage = symbolData.totalCoverage;
let statusClass, statusText;
if (coverage >= 95) {
statusClass = 'status-excellent';
statusText = 'Excellent';
} else if (coverage >= 80) {
statusClass = 'status-good';
statusText = 'Good';
} else if (coverage >= 50) {
statusClass = 'status-partial';
statusText = 'Partial';
} else {
statusClass = 'status-poor';
statusText = 'Poor';
}
// Format dates
const fromDate = symbolData.firstDate ? new Date(symbolData.firstDate).toLocaleDateString() : 'N/A';
const toDate = symbolData.lastDate ? new Date(symbolData.lastDate).toLocaleDateString() : 'N/A';
// Create interval badges
const intervalBadges = symbolData.intervals.map(i =>
`<span class="interval-badge">${i.interval}</span>`
).join('');
// Health indicator
const healthIndicator = symbolData.healthScore !== null
? `<span style="color: ${getHealthColor(symbolData.healthScore)}">❤ ${symbolData.healthScore}</span>`
: '';
row.innerHTML = `
<td><strong>${symbolData.symbol}</strong> ${healthIndicator}</td>
<td>${fromDate}</td>
<td>${toDate}</td>
<td>${intervalBadges}</td>
<td>
<div class="progress-bar">
<div class="progress-fill" style="width: ${coverage}%"></div>
</div>
<small>${coverage.toFixed(1)}% (${symbolData.totalMissing.toLocaleString()} missing)</small>
</td>
<td><span class="status-badge ${statusClass}">${statusText}</span></td>
<td>
<button class="action-button" onclick="showDetails('${symbolData.symbol}')">Details</button>
${symbolData.totalMissing > 0 ? `<button class="action-button secondary" onclick="smartFillGaps('${symbolData.symbol}')">Smart Fill</button>` : ''}
</td>
`;
tbody.appendChild(row);
});
}
function getHealthColor(score) {
if (score >= 90) return '#48bb78';
if (score >= 70) return '#4299e1';
if (score >= 50) return '#ed8936';
return '#f56565';
}
async function showDetails(symbol) {
const modal = document.getElementById('gapModal');
const modalTitle = document.getElementById('modalTitle');
const modalBody = document.getElementById('modalBody');
modalTitle.textContent = `${symbol} - Gap Details & Analytics`;
modalBody.innerHTML = '<div class="loading"><div class="spinner"></div><p>Loading details...</p></div>';
modal.style.display = 'block';
// Get data for this symbol
const symbolData = gapData.filter(item => item.symbol === symbol);
let detailsHTML = '';
for (const item of symbolData) {
const coverage = item.coverage_percent || 0;
const statusClass = coverage >= 95 ? 'status-excellent' : coverage >= 80 ? 'status-good' : coverage >= 50 ? 'status-partial' : 'status-poor';
// Get progress and health data
const progressData = await fetchProgress(symbol, item.interval);
const healthData = await fetchHealth(symbol, item.interval);
const prioritizedGaps = await fetchPrioritizedGaps(symbol, item.interval);
detailsHTML += `
<div class="gap-item">
<h3>${item.interval} <span class="status-badge ${statusClass}">${coverage.toFixed(1)}%</span></h3>
${healthData ? `
<div style="background: #fff5f5; padding: 10px; border-radius: 4px; margin: 10px 0; border-left: 3px solid ${getHealthColor(healthData.health_score)}">
<strong>Health Score: ${healthData.health_score}/100</strong> (${healthData.status})
${healthData.issues.length > 0 ? `
<ul style="margin: 5px 0 0 20px; font-size: 12px;">
${healthData.issues.map(issue => `<li>${issue.message}</li>`).join('')}
</ul>
` : '<p style="margin: 5px 0 0 0; font-size: 12px; color: #48bb78;">✓ No issues detected</p>'}
</div>
` : ''}
${progressData && progressData.missing_records > 0 ? `
<div style="background: #ebf8ff; padding: 10px; border-radius: 4px; margin: 10px 0;">
<strong>⏱ Estimated Time to Complete:</strong> ${progressData.estimated_time_human}
<small style="display: block; margin-top: 5px; color: #718096;">
${progressData.missing_records.toLocaleString()} records remaining
</small>
</div>
` : ''}
<div class="gap-info">
<div class="gap-info-item">
<span class="gap-label">First Record:</span>
<span class="gap-value">${item.first_record ? new Date(item.first_record).toLocaleString() : 'N/A'}</span>
</div>
<div class="gap-info-item">
<span class="gap-label">Last Record:</span>
<span class="gap-value">${item.last_record ? new Date(item.last_record).toLocaleString() : 'N/A'}</span>
</div>
<div class="gap-info-item">
<span class="gap-label">Total Records:</span>
<span class="gap-value">${(item.total_records || 0).toLocaleString()}</span>
</div>
<div class="gap-info-item">
<span class="gap-label">Expected Records:</span>
<span class="gap-value">${(item.expected_records || 0).toLocaleString()}</span>
</div>
<div class="gap-info-item">
<span class="gap-label">Missing Records:</span>
<span class="gap-value">${(item.missing_records || 0).toLocaleString()}</span>
</div>
<div class="gap-info-item">
<span class="gap-label">Number of Gaps:</span>
<span class="gap-value">${(item.gaps || []).length}</span>
</div>
</div>
`;
if (prioritizedGaps && prioritizedGaps.length > 0) {
detailsHTML += '<h4 style="margin-top: 15px; color: #2d3748;">🎯 Priority Gaps (Smartly Sorted):</h4>';
prioritizedGaps.slice(0, 10).forEach((gap, idx) => {
const priorityColor = gap.priority_score > 150 ? '#48bb78' : gap.priority_score > 100 ? '#4299e1' : '#ed8936';
detailsHTML += `
<div style="background: #edf2f7; padding: 10px; margin: 5px 0; border-radius: 4px; font-size: 13px; border-left: 3px solid ${priorityColor}">
<strong>#${idx + 1} Priority:</strong> ${gap.priority_score.toFixed(1)} |
<strong>Age:</strong> ${gap.days_old} days<br>
<strong>Gap:</strong> ${new Date(gap.gap_start).toLocaleString()} → ${new Date(gap.gap_end).toLocaleString()}<br>
<strong>Missing:</strong> ${gap.missing_candles} candles (${gap.duration_hours}h)
</div>
`;
});
if (prioritizedGaps.length > 10) {
detailsHTML += `<p style="color: #718096; font-size: 13px; margin-top: 5px;">... and ${prioritizedGaps.length - 10} more gaps</p>`;
}
}
detailsHTML += '</div>';
}
modalBody.innerHTML = detailsHTML || '<p>No gap data available</p>';
}
async function fetchProgress(symbol, interval) {
try {
const response = await fetch(`/api/gaps/progress/${symbol}/${interval}`);
const result = await response.json();
return result.status === 'success' ? result.data : null;
} catch (error) {
console.error('Error fetching progress:', error);
return null;
}
}
async function fetchHealth(symbol, interval) {
try {
const response = await fetch(`/api/gaps/health/${symbol}/${interval}`);
const result = await response.json();
return result.status === 'success' ? result.data : null;
} catch (error) {
console.error('Error fetching health:', error);
return null;
}
}
async function fetchPrioritizedGaps(symbol, interval) {
try {
const response = await fetch(`/api/gaps/prioritized/${symbol}/${interval}`);
const result = await response.json();
return result.status === 'success' ? result.data : null;
} catch (error) {
console.error('Error fetching prioritized gaps:', error);
return null;
}
}
async function smartFillGaps(symbol) {
if (!confirm(`Smart fill will automatically prioritize and fill the most important gaps for ${symbol}. Continue?`)) {
return;
}
try {
const response = await fetch(`/api/gaps/smart-fill/${symbol}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' }
});
const result = await response.json();
if (result.status === 'success') {
alert(`Smart fill completed for ${symbol}!\\n\\n` +
result.data.map(r => `${r.interval}: ${r.gaps_filled}/${r.total_gaps} filled`).join('\\n'));
// Reload data
setTimeout(loadGapData, 3000);
} else {
alert('Error: ' + result.message);
}
} catch (error) {
alert('Error filling gaps: ' + error.message);
}
}
function closeModal() {
document.getElementById('gapModal').style.display = 'none';
}
function showError(message) {
document.getElementById('loadingIndicator').innerHTML = `
<p style="color: #e53e3e; font-weight: bold;">${message}</p>
`;
}
// Close modal when clicking outside
window.onclick = function(event) {
const modal = document.getElementById('gapModal');
if (event.target == modal) {
closeModal();
}
}
// Load data on page load
loadGapData();
// Auto-refresh every 30 seconds
setInterval(loadGapData, 30000);
</script>
</body>
</html>
"""