lewis/Market-Data-Downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add API endpoints for technical indicator coverage management

Browse Source 
- Add GET /api/indicators/coverage/{symbol}/{interval} to check coverage
- Add GET /api/indicators/coverage/all for system-wide coverage status
- Add POST /api/indicators/backfill/{symbol}/{interval} to backfill missing indicators
- Add POST /api/indicators/backfill-all for bulk backfill operations
- Add GET /api/indicators/missing/{symbol}/{interval} to list incomplete records
- Add GET /api/indicators/summary for aggregate coverage statistics
- Support configurable batch_size and min_coverage_threshold parameters
- Return detailed results including before/after coverage percentages
- Provide summary statistics with coverage ranges and lowest coverage pairs
- Enable UI integration for monitoring and managing indicator completeness

These endpoints expose the db.py indicator coverage methods through the
web API, allowing users to monitor and maintain 100% technical indicator
coverage across all trading pairs via the web interface.
This commit is contained in:
lewismac
2025-10-09 08:55:22 +01:00
parent e891cd7c15
commit 9db0c4ca80
1 changed files with 334 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

343
ui_routes.py
View File

@@ -1,19 +1,19 @@
#!/usr/bin/env python3
"""
ui_routes.py - API Endpoints and Route Handlers
Defines all FastAPI routes and business logic for API endpoints
"""
import asyncio
import logging
from datetime import datetime, timedelta, timezone
from typing import Dict, Any, Optional, List
from fastapi import HTTPException, Request
from fastapi.responses import HTMLResponse, JSONResponse
from dotenv import set_key, dotenv_values
# UI and models
@@ -101,10 +101,205 @@ class APIRoutes:
try:
status = await get_current_status(self.db_manager, self.data_collector, self.config)
return JSONResponse(content=serialize_for_json(status))
except Exception as e:
logger.error(f"Error getting stats: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=str(e))
# ---------------------------
# Indicator Coverage (NEW)
# ---------------------------
@self.app.get("/api/indicators/coverage/{symbol}/{interval}")
async def get_indicator_coverage(symbol: str, interval: str):
"""Get technical indicator coverage status for a symbol/interval"""
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
coverage = await self.db_manager.check_indicator_coverage(symbol.upper(), interval)
return _ok(coverage)
except Exception as e:
logger.error(f"Error checking indicator coverage: {e}", exc_info=True)
return _err(str(e), 500)
@self.app.get("/api/indicators/coverage/all")
async def get_all_indicator_coverage():
"""Get indicator coverage status for all symbol/interval combinations"""
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
logger.info("Fetching indicator coverage for all pairs")
coverage_list = await self.db_manager.get_all_indicator_coverage_status()
logger.info(f"Retrieved coverage for {len(coverage_list)} combinations")
return _ok(coverage_list)
except Exception as e:
logger.error(f"Error getting all indicator coverage: {e}", exc_info=True)
return _err(str(e), 500)
@self.app.post("/api/indicators/backfill/{symbol}/{interval}")
async def backfill_indicators(symbol: str, interval: str, request: Request):
"""Backfill missing technical indicators for a symbol/interval"""
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
body = await request.json() if request.headers.get('content-type') == 'application/json' else {}
batch_size = body.get('batch_size', 200)
logger.info(f"Starting indicator backfill for {symbol} {interval} (batch_size={batch_size})")
result = await self.db_manager.backfill_missing_indicators(
symbol.upper(), interval, batch_size=batch_size
)
logger.info(f"Indicator backfill completed: {result}")
return _ok(result)
except Exception as e:
logger.error(f"Error backfilling indicators: {e}", exc_info=True)
return _err(str(e), 500)
@self.app.post("/api/indicators/backfill-all")
async def backfill_all_indicators(request: Request):
"""Backfill missing indicators for all symbol/interval combinations with incomplete coverage"""
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
body = await request.json() if request.headers.get('content-type') == 'application/json' else {}
batch_size = body.get('batch_size', 200)
min_coverage_threshold = body.get('min_coverage_threshold', 99.9)
logger.info("Starting indicator backfill for all pairs with incomplete coverage")
# Get all coverage status
coverage_list = await self.db_manager.get_all_indicator_coverage_status()
# Filter to only those needing backfill
needs_backfill = [
c for c in coverage_list
if c['coverage_percent'] < min_coverage_threshold
]
logger.info(f"Found {len(needs_backfill)} combinations needing backfill")
results = []
for coverage in needs_backfill:
symbol = coverage['symbol']
interval = coverage['interval']
try:
result = await self.db_manager.backfill_missing_indicators(
symbol, interval, batch_size=batch_size
)
results.append({
'symbol': symbol,
'interval': interval,
'status': result['status'],
'coverage_before': result.get('coverage_before', 0),
'coverage_after': result.get('coverage_after', 0),
'indicators_added': result.get('indicators_added', 0)
})
logger.info(
f"Backfilled {symbol} {interval}: "
f"{result.get('coverage_before', 0):.2f}% → {result.get('coverage_after', 0):.2f}%"
)
# Small delay to avoid overwhelming the database
await asyncio.sleep(0.5)
except Exception as e:
logger.error(f"Error backfilling {symbol} {interval}: {e}")
results.append({
'symbol': symbol,
'interval': interval,
'status': 'error',
'error': str(e)
})
summary = {
'total_checked': len(coverage_list),
'needed_backfill': len(needs_backfill),
'processed': len(results),
'successful': len([r for r in results if r.get('status') == 'success']),
'failed': len([r for r in results if r.get('status') == 'error']),
'results': results
}
logger.info(f"Bulk indicator backfill complete: {summary['successful']}/{summary['processed']} successful")
return _ok(summary)
except Exception as e:
logger.error(f"Error in bulk indicator backfill: {e}", exc_info=True)
return _err(str(e), 500)
@self.app.get("/api/indicators/missing/{symbol}/{interval}")
async def get_missing_indicators(symbol: str, interval: str):
"""Get list of OHLCV records missing technical indicators"""
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
missing = await self.db_manager.get_ohlcv_missing_indicators(
symbol.upper(), interval, limit=100
)
return _ok({
'symbol': symbol.upper(),
'interval': interval,
'missing_count': len(missing),
'missing_records': missing
})
except Exception as e:
logger.error(f"Error getting missing indicators: {e}", exc_info=True)
return _err(str(e), 500)
@self.app.get("/api/indicators/summary")
async def get_indicators_summary():
"""Get summary of indicator coverage across all symbols"""
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
coverage_list = await self.db_manager.get_all_indicator_coverage_status()
# Calculate summary statistics
total_combinations = len(coverage_list)
complete_combinations = len([c for c in coverage_list if c['coverage_percent'] >= 99.9])
incomplete_combinations = total_combinations - complete_combinations
avg_coverage = sum(c['coverage_percent'] for c in coverage_list) / total_combinations if total_combinations > 0 else 0
# Group by coverage ranges
coverage_ranges = {
'100%': len([c for c in coverage_list if c['coverage_percent'] >= 99.9]),
'90-99%': len([c for c in coverage_list if 90 <= c['coverage_percent'] < 99.9]),
'50-89%': len([c for c in coverage_list if 50 <= c['coverage_percent'] < 90]),
'0-49%': len([c for c in coverage_list if c['coverage_percent'] < 50]),
}
summary = {
'total_combinations': total_combinations,
'complete_combinations': complete_combinations,
'incomplete_combinations': incomplete_combinations,
'average_coverage': round(avg_coverage, 2),
'coverage_ranges': coverage_ranges,
'lowest_coverage': sorted(coverage_list, key=lambda x: x['coverage_percent'])[:10] if coverage_list else []
}
return _ok(summary)
except Exception as e:
logger.error(f"Error getting indicators summary: {e}", exc_info=True)
return _err(str(e), 500)
# ---------------------------
# Gaps and Coverage
# ---------------------------
@@ -116,10 +311,13 @@ class APIRoutes:
if not self.db_manager:
logger.error("Database manager not initialized")
return _err("Database not initialized", 500)
logger.info("Fetching gap status for all pairs")
status = await self.db_manager.get_all_pairs_gap_status()
logger.info(f"Retrieved gap status for {len(status)} pair-interval combinations")
return _ok(status)
except Exception as e:
logger.error(f"Error getting all pairs gaps: {e}", exc_info=True)
return _err(str(e), 500)
@@ -133,8 +331,10 @@ class APIRoutes:
sym = symbol.upper()
gap_info = await self.db_manager.detect_gaps(sym, interval)
end_date = datetime.utcnow()
start_date = end_date - timedelta(days=90)
daily_coverage = await self.db_manager.get_data_coverage_by_day(sym, interval, start_date, end_date)
data = {
@@ -144,7 +344,9 @@ class APIRoutes:
"gaps": gap_info.get('gaps', []),
"daily_coverage": daily_coverage.get('daily_coverage', []),
}
return _ok(data)
except Exception as e:
logger.error(f"Error getting gap details: {e}", exc_info=True)
return _err(str(e), 500)
@@ -165,8 +367,10 @@ class APIRoutes:
raise HTTPException(status_code=500, detail="Database not initialized")
result = await self.db_manager.fill_gaps_intelligently(symbol.upper(), interval, max_attempts)
logger.info(f"Intelligent gap fill completed: {result}")
return _ok(result)
except Exception as e:
logger.error(f"Error in intelligent gap fill: {e}", exc_info=True)
return _err(str(e), 500)
@@ -177,8 +381,10 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
prioritized = await self.db_manager.get_prioritized_gaps(symbol.upper(), interval)
return _ok(prioritized)
except Exception as e:
logger.error(f"Error getting prioritized gaps: {e}", exc_info=True)
return _err(str(e), 500)
@@ -189,8 +395,10 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
progress = await self.db_manager.get_gap_fill_progress(symbol.upper(), interval)
return _ok(progress)
except Exception as e:
logger.error(f"Error getting gap progress: {e}", exc_info=True)
return _err(str(e), 500)
@@ -201,8 +409,10 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
health = await self.db_manager.check_data_health(symbol.upper(), interval)
return _ok(health)
except Exception as e:
logger.error(f"Error checking data health: {e}", exc_info=True)
return _err(str(e), 500)
@@ -219,21 +429,30 @@ class APIRoutes:
intervals = cfg.get('collection', {}).get('candle_intervals', ['1m', '5m', '15m', '1h', '4h', '1d'])
results: List[Dict[str, Any]] = []
for interval in intervals:
prioritized = await self.db_manager.get_prioritized_gaps(symbol.upper(), interval)
if not prioritized:
continue
filled = 0
for gap in prioritized[:5]:
for gap in prioritized[:5]: # Only fill top 5 gaps
if gap.get('missing_candles', 0) <= 100:
try:
await self.db_manager.fill_gaps_intelligently(symbol.upper(), interval, max_attempts=3)
filled += 1
except Exception as e:
logger.error(f"Error filling gap: {e}")
results.append({'interval': interval, 'gaps_filled': filled, 'total_gaps': len(prioritized)})
return JSONResponse(content={"status": "success", "message": f"Smart fill completed for {symbol}", "data": results})
return JSONResponse(content={
"status": "success",
"message": f"Smart fill completed for {symbol}",
"data": results
})
except Exception as e:
logger.error(f"Error in smart fill: {e}", exc_info=True)
return _err(str(e), 500)
@@ -247,6 +466,7 @@ class APIRoutes:
gap_start = datetime.fromisoformat(request.gap_start)
gap_end = datetime.fromisoformat(request.gap_end)
gap_start = _tz_aware(gap_start)
gap_end = _tz_aware(gap_end)
@@ -256,8 +476,10 @@ class APIRoutes:
gap_end,
[request.interval],
)
logger.info(f"Gap filled for {request.symbol} {request.interval}")
return JSONResponse(content={"status": "success", "message": "Gap filled successfully"})
except Exception as e:
logger.error(f"Error filling gap: {e}", exc_info=True)
return _err(str(e), 500)
@@ -268,13 +490,20 @@ class APIRoutes:
try:
if not self.data_collector:
raise HTTPException(status_code=500, detail="Data collector not initialized")
result = await self.data_collector.auto_fill_gaps(
request.symbol.upper(),
request.intervals,
request.fill_genuine_gaps,
)
logger.info(f"Auto gap fill completed for {request.symbol}: {result}")
return JSONResponse(content={"status": "success", "message": f"Filled gaps for {request.symbol}", "result": serialize_for_json(result)})
return JSONResponse(content={
"status": "success",
"message": f"Filled gaps for {request.symbol}",
"result": serialize_for_json(result)
})
except Exception as e:
logger.error(f"Error in auto gap fill: {e}", exc_info=True)
return _err(str(e), 500)
@@ -285,8 +514,10 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
summary = await self.db_manager.get_all_gaps_summary()
return _ok(summary)
except Exception as e:
logger.error(f"Error getting gaps summary: {e}", exc_info=True)
return _err(str(e), 500)
@@ -297,8 +528,10 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
status = await self.db_manager.get_gap_fill_status(symbol.upper(), interval)
return _ok(status)
except Exception as e:
logger.error(f"Error getting gap status: {e}", exc_info=True)
return _err(str(e), 500)
@@ -309,8 +542,10 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
gaps = await self.db_manager.detect_gaps(symbol.upper(), interval)
return JSONResponse(content={"status": "success", "gaps": serialize_for_json(gaps)})
except Exception as e:
logger.error(f"Error detecting gaps: {e}", exc_info=True)
return _err(str(e), 500)
@@ -321,12 +556,15 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
gap_config = self.config.get('gap_filling', {})
max_consecutive = int(gap_config.get('max_consecutive_empty_candles', 5))
lookback = int(gap_config.get('averaging_lookback_candles', 10))
filled_count = await self.db_manager.fill_genuine_gaps_with_averages(
symbol.upper(), interval, max_consecutive, lookback
)
logger.info(f"Filled {filled_count} genuine gaps for {symbol} {interval}")
return JSONResponse(
content={
@@ -335,6 +573,7 @@ class APIRoutes:
"filled_count": filled_count,
}
)
except Exception as e:
logger.error(f"Error filling genuine gaps: {e}", exc_info=True)
return _err(str(e), 500)
@@ -350,9 +589,11 @@ class APIRoutes:
if not self.db_manager:
logger.error("Database manager not initialized")
return JSONResponse(content={"status": "error", "symbols": []}, status_code=500)
symbols = await self.db_manager.get_available_symbols()
logger.info(f"Retrieved {len(symbols)} symbols from database")
return JSONResponse(content={"status": "success", "symbols": symbols})
except Exception as e:
logger.error(f"Error getting symbols: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "symbols": []}, status_code=500)
@@ -364,15 +605,21 @@ class APIRoutes:
if not self.db_manager:
logger.error("Database manager not initialized")
return _err("Database not initialized", 500)
logger.info(f"Price trends request for {symbol}")
data = await self.db_manager.get_current_price_and_trends_with_volume(symbol.upper())
if not data:
logger.warning(f"No price data found for {symbol}")
return _err(f"No data found for {symbol}. Please start data collection first.", 404)
pair_config = next((p for p in self.config.get('trading_pairs', []) if p['symbol'] == symbol.upper()), None)
data['enabled'] = pair_config.get('enabled', False) if pair_config else False
logger.info(f"Returning price trends for {symbol}: price={data.get('current_price')}")
return _ok(data)
except Exception as e:
logger.error(f"Error getting price trends: {e}", exc_info=True)
return _err(f"Error retrieving price trends: {str(e)}", 500)
@@ -387,12 +634,16 @@ class APIRoutes:
try:
if not self.data_collector:
raise HTTPException(status_code=500, detail="Data collector not initialized")
if self.state_manager.get("is_collecting", False):
return JSONResponse(content={"status": "info", "message": "Collection already running"})
await self.data_collector.start_continuous_collection()
self.state_manager.update(is_collecting=True)
logger.info("Collection started via API")
return JSONResponse(content={"status": "success", "message": "Collection started"})
except Exception as e:
logger.error(f"Error starting collection: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -403,12 +654,16 @@ class APIRoutes:
try:
if not self.data_collector:
raise HTTPException(status_code=500, detail="Data collector not initialized")
if not self.state_manager.get("is_collecting", False):
return JSONResponse(content={"status": "info", "message": "Collection not running"})
await self.data_collector.stop_continuous_collection()
self.state_manager.update(is_collecting=False)
logger.info("Collection stopped via API")
return JSONResponse(content={"status": "success", "message": "Collection stopped"})
except Exception as e:
logger.error(f"Error stopping collection: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -423,6 +678,7 @@ class APIRoutes:
try:
cfg = load_config()
return JSONResponse(content=serialize_for_json(cfg))
except Exception as e:
logger.error(f"Error getting config: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=str(e))
@@ -432,9 +688,11 @@ class APIRoutes:
"""Update configuration - accepts raw JSON body"""
try:
body = await request.json()
logger.info(f"Received config update keys: {list(body.keys())}")
current_config = load_config()
# Deep merge/replace top-level keys
for key, value in body.items():
if key in current_config and isinstance(current_config[key], dict) and isinstance(value, dict):
@@ -443,10 +701,13 @@ class APIRoutes:
current_config[key] = value
save_config(current_config)
self.config.clear()
self.config.update(current_config)
logger.info("Configuration updated successfully")
return JSONResponse(content={"status": "success", "message": "Configuration updated"})
except Exception as e:
logger.error(f"Error updating config: {e}", exc_info=True)
return _err(str(e), 500)
@@ -459,22 +720,28 @@ class APIRoutes:
return JSONResponse(content={"status": "error", "message": "Invalid symbol format"}, status_code=400)
cfg = load_config()
existing = [p for p in cfg.get('trading_pairs', []) if p['symbol'] == pair.symbol.upper()]
if existing:
return JSONResponse(content={"status": "error", "message": "Trading pair already exists"}, status_code=409)
record_from_date = pair.record_from_date or cfg.get('collection', {}).get('default_record_from_date', '2020-01-01T00:00:00Z')
cfg.setdefault('trading_pairs', []).append({
'symbol': pair.symbol.upper(),
'enabled': True,
'priority': pair.priority,
'record_from_date': record_from_date,
})
save_config(cfg)
self.config.clear()
self.config.update(cfg)
logger.info(f"Added trading pair: {pair.symbol}")
return JSONResponse(content={"status": "success", "message": f"Added {pair.symbol}"})
except Exception as e:
logger.error(f"Error adding trading pair: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -485,6 +752,7 @@ class APIRoutes:
try:
update = await request.json()
logger.info(f"Updating trading pair {symbol}: {update}")
cfg = load_config()
pair_found = False
@@ -496,6 +764,7 @@ class APIRoutes:
pair['priority'] = int(update['priority'])
if 'record_from_date' in update:
pair['record_from_date'] = update['record_from_date']
pair_found = True
break
@@ -503,10 +772,13 @@ class APIRoutes:
return JSONResponse(content={"status": "error", "message": "Trading pair not found"}, status_code=404)
save_config(cfg)
self.config.clear()
self.config.update(cfg)
logger.info(f"Updated trading pair: {symbol}")
return JSONResponse(content={"status": "success", "message": f"Updated {symbol}"})
except Exception as e:
logger.error(f"Error updating trading pair: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -516,6 +788,7 @@ class APIRoutes:
"""Remove a trading pair"""
try:
cfg = load_config()
original_count = len(cfg.get('trading_pairs', []))
cfg['trading_pairs'] = [p for p in cfg.get('trading_pairs', []) if p['symbol'] != symbol.upper()]
@@ -523,10 +796,13 @@ class APIRoutes:
return JSONResponse(content={"status": "error", "message": "Trading pair not found"}, status_code=404)
save_config(cfg)
self.config.clear()
self.config.update(cfg)
logger.info(f"Removed trading pair: {symbol}")
return JSONResponse(content={"status": "success", "message": f"Removed {symbol}"})
except Exception as e:
logger.error(f"Error removing trading pair: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -537,17 +813,26 @@ class APIRoutes:
try:
cfg = load_config()
enabled_indicators = cfg.setdefault('technical_indicators', {}).setdefault('enabled', [])
if indicator_name in enabled_indicators:
enabled_indicators.remove(indicator_name)
action = "disabled"
else:
enabled_indicators.append(indicator_name)
action = "enabled"
save_config(cfg)
self.config.clear()
self.config.update(cfg)
logger.info(f"Indicator {indicator_name} {action}")
return JSONResponse(content={"status": "success", "message": f"Indicator {indicator_name} {action}", "enabled": indicator_name in enabled_indicators})
return JSONResponse(content={
"status": "success",
"message": f"Indicator {indicator_name} {action}",
"enabled": indicator_name in enabled_indicators
})
except Exception as e:
logger.error(f"Error toggling indicator: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -558,20 +843,26 @@ class APIRoutes:
try:
body = await request.json()
periods = body.get('periods')
if periods is None:
return JSONResponse(content={"status": "error", "message": "Missing 'periods' in request"}, status_code=400)
cfg = load_config()
periods_cfg = cfg.setdefault('technical_indicators', {}).setdefault('periods', {})
if indicator_name not in periods_cfg:
return JSONResponse(content={"status": "error", "message": f"Unknown indicator: {indicator_name}"}, status_code=404)
periods_cfg[indicator_name] = periods
save_config(cfg)
self.config.clear()
self.config.update(cfg)
logger.info(f"Updated {indicator_name} periods to {periods}")
return JSONResponse(content={"status": "success", "message": f"Updated {indicator_name} periods"})
except Exception as e:
logger.error(f"Error updating indicator periods: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -589,13 +880,21 @@ class APIRoutes:
return JSONResponse(content={"status": "error", "message": "Database not initialized"}, status_code=500)
logger.info(f"Chart data request: symbol={request.symbol}, interval={request.interval}, limit={request.limit}")
data = await self.db_manager.get_recent_candles(request.symbol.upper(), request.interval, request.limit)
logger.info(f"Retrieved {len(data) if data else 0} candles from database")
if not data:
logger.warning(f"No data found for {request.symbol} at {request.interval}")
return JSONResponse(content={"status": "error", "message": f"No data found for {request.symbol} at {request.interval}. Please start data collection or download historical data first."}, status_code=404)
return JSONResponse(content={
"status": "error",
"message": f"No data found for {request.symbol} at {request.interval}. Please start data collection or download historical data first."
}, status_code=404)
logger.info(f"Returning {len(data)} candles for {request.symbol}")
return JSONResponse(content={"status": "success", "data": data})
except Exception as e:
logger.error(f"Error getting chart data: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": f"Error retrieving chart data: {str(e)}"}, status_code=500)
@@ -609,32 +908,42 @@ class APIRoutes:
start_date = datetime.fromisoformat(request.start_date)
end_date = datetime.fromisoformat(request.end_date) if request.end_date else datetime.utcnow()
start_date = _tz_aware(start_date)
end_date = _tz_aware(end_date)
intervals = request.intervals or ['1h', '4h', '1d']
results = []
results = []
for symbol in request.symbols:
try:
symu = symbol.upper()
# Initialize progress for UI
self.data_collector.download_progress[symu] = {
'status': 'pending',
'intervals': {i: {'status': 'pending', 'records': 0} for i in intervals},
'start_time': datetime.now(timezone.utc).isoformat(),
}
# Spawn task
task = asyncio.create_task(
self.data_collector.bulk_download_historical_data(symu, start_date, end_date, intervals)
)
results.append({'symbol': symu, 'status': 'started', 'intervals': intervals})
logger.info(f"Bulk download started for {symbol}")
except Exception as ie:
logger.error(f"Error starting bulk download for {symbol}: {ie}")
results.append({'symbol': symu, 'status': 'error', 'error': str(ie)})
return JSONResponse(content={"status": "success", "message": f"Bulk download started for {len(request.symbols)} symbol(s)", "results": results})
return JSONResponse(content={
"status": "success",
"message": f"Bulk download started for {len(request.symbols)} symbol(s)",
"results": results
})
except Exception as e:
logger.error(f"Error starting bulk download: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -645,8 +954,10 @@ class APIRoutes:
try:
if not self.data_collector:
return JSONResponse(content={"status": "error", "message": "Data collector not initialized"}, status_code=500)
progress = await self.data_collector.get_download_progress()
return JSONResponse(content={"status": "success", "downloads": serialize_for_json(progress)})
except Exception as e:
logger.error(f"Error getting download progress: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -660,11 +971,14 @@ class APIRoutes:
"""Get environment variables"""
try:
env_vars = dotenv_values('variables.env') or {}
safe_vars = {
k: ('***' if any(s in k.upper() for s in ['SECRET', 'KEY', 'PASSWORD', 'TOKEN']) else v)
for k, v in env_vars.items()
}
return JSONResponse(content=safe_vars)
except Exception as e:
logger.error(f"Error getting env vars: {e}", exc_info=True)
raise HTTPException(status_code=500, detail=str(e))
@@ -675,11 +989,15 @@ class APIRoutes:
try:
key_upper = env_update.key.upper()
display_value = env_update.value if not any(s in key_upper for s in ['PASSWORD', 'SECRET', 'KEY', 'TOKEN']) else '***'
logger.info(f"Updating env var: {env_update.key} = {display_value}")
set_key('variables.env', env_update.key, env_update.value)
reload_env_vars('variables.env')
logger.info(f"Updated and reloaded env var: {env_update.key}")
return JSONResponse(content={"status": "success", "message": f"Updated {env_update.key}"})
except Exception as e:
logger.error(f"Error updating env var: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -694,12 +1012,17 @@ class APIRoutes:
lines = f.readlines()
except FileNotFoundError:
lines = []
new_lines = [line for line in lines if not line.startswith(f"{key}=")]
with open('variables.env', 'w', encoding='utf-8') as f:
f.writelines(new_lines)
reload_env_vars('variables.env')
logger.info(f"Deleted env var: {key}")
return JSONResponse(content={"status": "success", "message": f"Deleted {key}"})
except Exception as e:
logger.error(f"Error deleting env var: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)
@@ -714,8 +1037,10 @@ class APIRoutes:
try:
if not self.db_manager:
raise HTTPException(status_code=500, detail="Database not initialized")
stats = await self.db_manager.get_detailed_statistics()
return JSONResponse(content={"status": "success", "stats": serialize_for_json(stats)})
except Exception as e:
logger.error(f"Error getting database stats: {e}", exc_info=True)
return JSONResponse(content={"status": "error", "message": str(e)}, status_code=500)