diff --git a/scripts/backfill.ts b/scripts/backfill.ts new file mode 100644 index 0000000..a13072a --- /dev/null +++ b/scripts/backfill.ts @@ -0,0 +1,424 @@ +/** + * Historical data backfill script. + * + * Populates 6 months of historical data from EIA and FRED into Postgres. + * Idempotent — safe to re-run; existing records are updated, not duplicated. + * + * Usage: bun run scripts/backfill.ts + */ + +import 'dotenv/config'; + +import { PrismaPg } from '@prisma/adapter-pg'; +import { PrismaClient } from '../src/generated/prisma/client.js'; + +import * as eia from '../src/lib/api/eia.js'; +import { getFuelTypeData, getRegionData } from '../src/lib/api/eia.js'; +import * as fred from '../src/lib/api/fred.js'; +import type { RegionCode } from '../src/lib/schemas/electricity.js'; + +const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL }); +const prisma = new PrismaClient({ adapter }); + +const ALL_REGIONS: RegionCode[] = ['PJM', 'ERCOT', 'CAISO', 'NYISO', 'ISONE', 'MISO', 'SPP']; + +const SIX_MONTHS_MS = 6 * 30 * 24 * 60 * 60 * 1000; + +function sixMonthsAgoIso(): string { + return new Date(Date.now() - SIX_MONTHS_MS).toISOString().slice(0, 10); +} + +function todayIso(): string { + return new Date().toISOString().slice(0, 10); +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +function log(msg: string): void { + const ts = new Date().toISOString().slice(11, 19); + console.log(`[${ts}] ${msg}`); +} + +// --------------------------------------------------------------------------- +// Electricity demand backfill +// --------------------------------------------------------------------------- + +async function backfillElectricity(): Promise { + log('=== Backfilling electricity demand data ==='); + + const gridRegions = await prisma.gridRegion.findMany({ + select: { id: true, code: true }, + }); + const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id])); + + const start = sixMonthsAgoIso(); + const end = todayIso(); + + for (const regionCode of ALL_REGIONS) { + const regionId = regionIdByCode.get(regionCode); + if (!regionId) { + log(` SKIP ${regionCode} — no grid_region row found`); + continue; + } + + log(` Fetching demand for ${regionCode}...`); + try { + const demandData = await getRegionData(regionCode, 'D', { start, end }); + const validPoints = demandData.filter((p): p is typeof p & { valueMw: number } => p.valueMw !== null); + + if (validPoints.length === 0) { + log(` ${regionCode}: 0 valid data points`); + continue; + } + + // Check existing records to decide create vs update + const timestamps = validPoints.map(p => p.timestamp); + const existing = await prisma.electricityPrice.findMany({ + where: { regionId, timestamp: { in: timestamps } }, + select: { id: true, timestamp: true }, + }); + const existingByTime = new Map(existing.map(e => [e.timestamp.getTime(), e.id])); + + const toCreate: Array<{ + regionId: string; + priceMwh: number; + demandMw: number; + timestamp: Date; + source: string; + }> = []; + const toUpdate: Array<{ id: string; demandMw: number }> = []; + + for (const point of validPoints) { + const existingId = existingByTime.get(point.timestamp.getTime()); + if (existingId) { + toUpdate.push({ id: existingId, demandMw: point.valueMw }); + } else { + toCreate.push({ + regionId, + priceMwh: 0, // TODO: No real-time wholesale price available from EIA + demandMw: point.valueMw, + timestamp: point.timestamp, + source: 'EIA', + }); + } + } + + if (toCreate.length > 0) { + const result = await prisma.electricityPrice.createMany({ data: toCreate }); + log(` ${regionCode}: ${result.count} records inserted`); + } + + if (toUpdate.length > 0) { + // Batch updates in chunks of 100 to avoid transaction timeouts + const chunkSize = 100; + for (let i = 0; i < toUpdate.length; i += chunkSize) { + const chunk = toUpdate.slice(i, i + chunkSize); + await prisma.$transaction( + chunk.map(u => + prisma.electricityPrice.update({ + where: { id: u.id }, + data: { demandMw: u.demandMw, source: 'EIA' }, + }), + ), + ); + } + log(` ${regionCode}: ${toUpdate.length} records updated`); + } + + if (toCreate.length === 0 && toUpdate.length === 0) { + log(` ${regionCode}: no changes needed`); + } + } catch (err) { + log(` ERROR ${regionCode}: ${err instanceof Error ? err.message : String(err)}`); + } + + // Rate limit: 200ms between regions + await sleep(200); + } +} + +// --------------------------------------------------------------------------- +// Generation mix backfill +// --------------------------------------------------------------------------- + +async function backfillGeneration(): Promise { + log('=== Backfilling generation mix data ==='); + + const gridRegions = await prisma.gridRegion.findMany({ + select: { id: true, code: true }, + }); + const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id])); + + const start = sixMonthsAgoIso(); + const end = todayIso(); + + for (const regionCode of ALL_REGIONS) { + const regionId = regionIdByCode.get(regionCode); + if (!regionId) { + log(` SKIP ${regionCode} — no grid_region row found`); + continue; + } + + log(` Fetching generation mix for ${regionCode}...`); + try { + const fuelData = await getFuelTypeData(regionCode, { start, end }); + const validPoints = fuelData.filter((p): p is typeof p & { generationMw: number } => p.generationMw !== null); + + if (validPoints.length === 0) { + log(` ${regionCode}: 0 valid data points`); + continue; + } + + const timestamps = validPoints.map(p => p.timestamp); + const existing = await prisma.generationMix.findMany({ + where: { regionId, timestamp: { in: timestamps } }, + select: { id: true, timestamp: true, fuelType: true }, + }); + const existingKeys = new Map(existing.map(e => [`${e.fuelType}:${e.timestamp.getTime()}`, e.id])); + + const toCreate: Array<{ + regionId: string; + fuelType: string; + generationMw: number; + timestamp: Date; + }> = []; + const toUpdate: Array<{ id: string; generationMw: number }> = []; + + for (const point of validPoints) { + const key = `${point.fuelType}:${point.timestamp.getTime()}`; + const existingId = existingKeys.get(key); + if (existingId) { + toUpdate.push({ id: existingId, generationMw: point.generationMw }); + } else { + toCreate.push({ + regionId, + fuelType: point.fuelType, + generationMw: point.generationMw, + timestamp: point.timestamp, + }); + } + } + + if (toCreate.length > 0) { + const result = await prisma.generationMix.createMany({ data: toCreate }); + log(` ${regionCode}: ${result.count} generation records inserted`); + } + + if (toUpdate.length > 0) { + const chunkSize = 100; + for (let i = 0; i < toUpdate.length; i += chunkSize) { + const chunk = toUpdate.slice(i, i + chunkSize); + await prisma.$transaction( + chunk.map(u => + prisma.generationMix.update({ + where: { id: u.id }, + data: { generationMw: u.generationMw }, + }), + ), + ); + } + log(` ${regionCode}: ${toUpdate.length} generation records updated`); + } + + if (toCreate.length === 0 && toUpdate.length === 0) { + log(` ${regionCode}: no changes needed`); + } + } catch (err) { + log(` ERROR ${regionCode}: ${err instanceof Error ? err.message : String(err)}`); + } + + await sleep(200); + } +} + +// --------------------------------------------------------------------------- +// Commodity prices backfill +// --------------------------------------------------------------------------- + +interface CommodityRow { + commodity: string; + price: number; + unit: string; + timestamp: Date; + source: string; +} + +async function backfillCommodities(): Promise { + log('=== Backfilling commodity prices ==='); + + const start = sixMonthsAgoIso(); + const end = todayIso(); + const rows: CommodityRow[] = []; + + // EIA: Natural Gas (Henry Hub) + log(' Fetching EIA natural gas prices...'); + try { + const gasData = await eia.getNaturalGasPrice({ start, end }); + for (const p of gasData) { + if (p.price === null) continue; + rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source }); + } + log(` EIA natural gas: ${gasData.length} raw records`); + } catch (err) { + log(` ERROR EIA natural gas: ${err instanceof Error ? err.message : String(err)}`); + } + + await sleep(200); + + // EIA: WTI Crude + log(' Fetching EIA WTI crude prices...'); + try { + const oilData = await eia.getWTICrudePrice({ start, end }); + for (const p of oilData) { + if (p.price === null) continue; + rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source }); + } + log(` EIA WTI crude: ${oilData.length} raw records`); + } catch (err) { + log(` ERROR EIA WTI crude: ${err instanceof Error ? err.message : String(err)}`); + } + + await sleep(200); + + // FRED: Natural Gas (DHHNGSP) + log(' Fetching FRED natural gas prices...'); + const fredGas = await fred.getNaturalGasPrice(start, end); + if (fredGas.ok) { + for (const p of fredGas.data) { + rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source }); + } + log(` FRED natural gas: ${fredGas.data.length} records`); + } else { + log(` ERROR FRED natural gas: ${fredGas.error}`); + } + + await sleep(200); + + // FRED: WTI Crude (DCOILWTICO) + log(' Fetching FRED WTI crude prices...'); + const fredOil = await fred.getWTICrudePrice(start, end); + if (fredOil.ok) { + for (const p of fredOil.data) { + rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source }); + } + log(` FRED WTI crude: ${fredOil.data.length} records`); + } else { + log(` ERROR FRED WTI crude: ${fredOil.error}`); + } + + await sleep(200); + + // FRED: Coal (PCOALAUUSDM) + log(' Fetching FRED coal prices...'); + const fredCoal = await fred.getCoalPrice(start, end); + if (fredCoal.ok) { + for (const p of fredCoal.data) { + rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source }); + } + log(` FRED coal: ${fredCoal.data.length} records`); + } else { + log(` ERROR FRED coal: ${fredCoal.error}`); + } + + if (rows.length === 0) { + log(' No commodity data fetched'); + return; + } + + // Deduplicate: for same commodity + timestamp, prefer EIA over FRED + const deduped = new Map(); + for (const row of rows) { + const key = `${row.commodity}:${row.timestamp.getTime()}`; + const existing = deduped.get(key); + if (!existing || (existing.source === 'FRED' && row.source === 'EIA')) { + deduped.set(key, row); + } + } + const uniqueRows = [...deduped.values()]; + + // Upsert into database + const timestamps = uniqueRows.map(r => r.timestamp); + const commodities = [...new Set(uniqueRows.map(r => r.commodity))]; + + const existing = await prisma.commodityPrice.findMany({ + where: { commodity: { in: commodities }, timestamp: { in: timestamps } }, + select: { id: true, commodity: true, timestamp: true }, + }); + const existingKeys = new Map(existing.map(e => [`${e.commodity}:${e.timestamp.getTime()}`, e.id])); + + const toCreate: Array<{ commodity: string; price: number; unit: string; timestamp: Date; source: string }> = []; + const toUpdate: Array<{ id: string; price: number; unit: string; source: string }> = []; + + for (const row of uniqueRows) { + const key = `${row.commodity}:${row.timestamp.getTime()}`; + const existingId = existingKeys.get(key); + if (existingId) { + toUpdate.push({ id: existingId, price: row.price, unit: row.unit, source: row.source }); + } else { + toCreate.push({ + commodity: row.commodity, + price: row.price, + unit: row.unit, + timestamp: row.timestamp, + source: row.source, + }); + } + } + + if (toCreate.length > 0) { + const result = await prisma.commodityPrice.createMany({ data: toCreate }); + log(` Commodities: ${result.count} records inserted`); + } + + if (toUpdate.length > 0) { + const chunkSize = 100; + for (let i = 0; i < toUpdate.length; i += chunkSize) { + const chunk = toUpdate.slice(i, i + chunkSize); + await prisma.$transaction( + chunk.map(u => + prisma.commodityPrice.update({ + where: { id: u.id }, + data: { price: u.price, unit: u.unit, source: u.source }, + }), + ), + ); + } + log(` Commodities: ${toUpdate.length} records updated`); + } + + if (toCreate.length === 0 && toUpdate.length === 0) { + log(' Commodities: no changes needed'); + } +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + log('Starting historical backfill (6 months)...'); + log(`Date range: ${sixMonthsAgoIso()} to ${todayIso()}`); + log(''); + + await backfillElectricity(); + log(''); + + await backfillGeneration(); + log(''); + + await backfillCommodities(); + log(''); + + log('Backfill complete.'); +} + +main() + .catch(err => { + console.error('Backfill failed:', err); + process.exit(1); + }) + .finally(() => { + void prisma.$disconnect(); + }); diff --git a/src/actions/datacenters.ts b/src/actions/datacenters.ts new file mode 100644 index 0000000..c1e94ce --- /dev/null +++ b/src/actions/datacenters.ts @@ -0,0 +1,74 @@ +'use server'; + +import { findDatacentersInRegion, findNearbyDatacenters } from '@/generated/prisma/sql.js'; +import { prisma } from '@/lib/db.js'; +import { serialize } from '@/lib/superjson.js'; + +interface ActionSuccess { + ok: true; + data: ReturnType>; +} + +interface ActionError { + ok: false; + error: string; +} + +type ActionResult = ActionSuccess | ActionError; + +export async function fetchDatacenters(): Promise< + ActionResult< + Array<{ + id: string; + name: string; + operator: string; + capacityMw: number; + status: string; + yearOpened: number; + regionId: string; + createdAt: Date; + }> + > +> { + try { + const rows = await prisma.datacenter.findMany({ + orderBy: { capacityMw: 'desc' }, + }); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch datacenters: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} + +export async function fetchDatacentersInRegion( + regionCode: string, +): Promise> { + try { + const rows = await prisma.$queryRawTyped(findDatacentersInRegion(regionCode)); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch datacenters in region ${regionCode}: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} + +export async function fetchNearbyDatacenters( + lat: number, + lng: number, + radiusKm: number, +): Promise> { + try { + const rows = await prisma.$queryRawTyped(findNearbyDatacenters(lat, lng, radiusKm)); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch nearby datacenters: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} diff --git a/src/actions/demand.ts b/src/actions/demand.ts new file mode 100644 index 0000000..8da653e --- /dev/null +++ b/src/actions/demand.ts @@ -0,0 +1,63 @@ +'use server'; + +import { getDemandByRegion } from '@/generated/prisma/sql.js'; +import { prisma } from '@/lib/db.js'; +import { serialize } from '@/lib/superjson.js'; + +type TimeRange = '24h' | '7d' | '30d' | '90d' | '1y'; + +function timeRangeToStartDate(range: TimeRange): Date { + const now = new Date(); + const ms: Record = { + '24h': 24 * 60 * 60 * 1000, + '7d': 7 * 24 * 60 * 60 * 1000, + '30d': 30 * 24 * 60 * 60 * 1000, + '90d': 90 * 24 * 60 * 60 * 1000, + '1y': 365 * 24 * 60 * 60 * 1000, + }; + return new Date(now.getTime() - ms[range]); +} + +interface ActionSuccess { + ok: true; + data: ReturnType>; +} + +interface ActionError { + ok: false; + error: string; +} + +type ActionResult = ActionSuccess | ActionError; + +export async function fetchDemandByRegion( + regionCode: string, + timeRange: TimeRange = '30d', +): Promise> { + try { + const startDate = timeRangeToStartDate(timeRange); + const endDate = new Date(); + const rows = await prisma.$queryRawTyped(getDemandByRegion(startDate, endDate)); + const filtered = regionCode === 'ALL' ? rows : rows.filter(r => r.region_code === regionCode); + return { ok: true, data: serialize(filtered) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch demand data: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} + +export async function fetchRegionDemandSummary(): Promise> { + try { + const startDate = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); + const endDate = new Date(); + const rows = await prisma.$queryRawTyped(getDemandByRegion(startDate, endDate)); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch region demand summary: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} diff --git a/src/actions/generation.ts b/src/actions/generation.ts new file mode 100644 index 0000000..583017f --- /dev/null +++ b/src/actions/generation.ts @@ -0,0 +1,48 @@ +'use server'; + +import { getGenerationMix } from '@/generated/prisma/sql.js'; +import { prisma } from '@/lib/db.js'; +import { serialize } from '@/lib/superjson.js'; + +type TimeRange = '24h' | '7d' | '30d' | '90d' | '1y'; + +function timeRangeToStartDate(range: TimeRange): Date { + const now = new Date(); + const ms: Record = { + '24h': 24 * 60 * 60 * 1000, + '7d': 7 * 24 * 60 * 60 * 1000, + '30d': 30 * 24 * 60 * 60 * 1000, + '90d': 90 * 24 * 60 * 60 * 1000, + '1y': 365 * 24 * 60 * 60 * 1000, + }; + return new Date(now.getTime() - ms[range]); +} + +interface ActionSuccess { + ok: true; + data: ReturnType>; +} + +interface ActionError { + ok: false; + error: string; +} + +type ActionResult = ActionSuccess | ActionError; + +export async function fetchGenerationMix( + regionCode: string, + timeRange: TimeRange = '30d', +): Promise> { + try { + const startDate = timeRangeToStartDate(timeRange); + const endDate = new Date(); + const rows = await prisma.$queryRawTyped(getGenerationMix(regionCode, startDate, endDate)); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch generation mix: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} diff --git a/src/actions/prices.ts b/src/actions/prices.ts new file mode 100644 index 0000000..7b8af10 --- /dev/null +++ b/src/actions/prices.ts @@ -0,0 +1,97 @@ +'use server'; + +import { getLatestPrices, getPriceTrends, getRegionPriceHeatmap } from '@/generated/prisma/sql.js'; +import { prisma } from '@/lib/db.js'; +import { serialize } from '@/lib/superjson.js'; + +type TimeRange = '24h' | '7d' | '30d' | '90d' | '1y'; + +function timeRangeToStartDate(range: TimeRange): Date { + const now = new Date(); + const ms: Record = { + '24h': 24 * 60 * 60 * 1000, + '7d': 7 * 24 * 60 * 60 * 1000, + '30d': 30 * 24 * 60 * 60 * 1000, + '90d': 90 * 24 * 60 * 60 * 1000, + '1y': 365 * 24 * 60 * 60 * 1000, + }; + return new Date(now.getTime() - ms[range]); +} + +interface ActionSuccess { + ok: true; + data: ReturnType>; +} + +interface ActionError { + ok: false; + error: string; +} + +type ActionResult = ActionSuccess | ActionError; + +export async function fetchLatestPrices(): Promise> { + try { + const rows = await prisma.$queryRawTyped(getLatestPrices()); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch latest prices: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} + +export async function fetchPriceTrends( + regionCode: string, + timeRange: TimeRange = '30d', +): Promise> { + try { + const startDate = timeRangeToStartDate(timeRange); + const endDate = new Date(); + const rows = await prisma.$queryRawTyped(getPriceTrends(regionCode, startDate, endDate)); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch price trends: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} + +export async function fetchPriceHeatmapData(): Promise> { + try { + const rows = await prisma.$queryRawTyped(getRegionPriceHeatmap()); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch price heatmap data: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} + +export async function fetchLatestCommodityPrices(): Promise< + ActionResult< + Array<{ + commodity: string; + price: number; + unit: string; + timestamp: Date; + source: string; + }> + > +> { + try { + const rows = await prisma.commodityPrice.findMany({ + orderBy: { timestamp: 'desc' }, + distinct: ['commodity'], + }); + return { ok: true, data: serialize(rows) }; + } catch (err) { + return { + ok: false, + error: `Failed to fetch commodity prices: ${err instanceof Error ? err.message : String(err)}`, + }; + } +} diff --git a/src/app/api/ingest/commodities/route.ts b/src/app/api/ingest/commodities/route.ts new file mode 100644 index 0000000..8ef90b6 --- /dev/null +++ b/src/app/api/ingest/commodities/route.ts @@ -0,0 +1,198 @@ +import { NextResponse, type NextRequest } from 'next/server.js'; + +import * as eia from '@/lib/api/eia.js'; +import * as fred from '@/lib/api/fred.js'; +import { prisma } from '@/lib/db.js'; + +interface IngestionStats { + inserted: number; + updated: number; + errors: number; +} + +interface CommodityRow { + commodity: string; + price: number; + unit: string; + timestamp: Date; + source: string; +} + +async function fetchAllCommodities(start?: string, end?: string): Promise<{ rows: CommodityRow[]; errors: number }> { + const rows: CommodityRow[] = []; + let errors = 0; + + // EIA: Natural Gas + try { + const gasData = await eia.getNaturalGasPrice({ start, end }); + for (const p of gasData) { + if (p.price === null) continue; + rows.push({ + commodity: p.commodity, + price: p.price, + unit: p.unit, + timestamp: p.timestamp, + source: p.source, + }); + } + } catch (err) { + console.error('Failed to fetch EIA natural gas:', err); + errors++; + } + + // EIA: WTI Crude + try { + const oilData = await eia.getWTICrudePrice({ start, end }); + for (const p of oilData) { + if (p.price === null) continue; + rows.push({ + commodity: p.commodity, + price: p.price, + unit: p.unit, + timestamp: p.timestamp, + source: p.source, + }); + } + } catch (err) { + console.error('Failed to fetch EIA WTI crude:', err); + errors++; + } + + // FRED: Natural Gas (DHHNGSP) + const fredGas = await fred.getNaturalGasPrice(start, end); + if (fredGas.ok) { + for (const p of fredGas.data) { + rows.push({ + commodity: p.commodity, + price: p.price, + unit: p.unit, + timestamp: p.timestamp, + source: p.source, + }); + } + } else { + console.error('Failed to fetch FRED natural gas:', fredGas.error); + errors++; + } + + // FRED: WTI Crude (DCOILWTICO) + const fredOil = await fred.getWTICrudePrice(start, end); + if (fredOil.ok) { + for (const p of fredOil.data) { + rows.push({ + commodity: p.commodity, + price: p.price, + unit: p.unit, + timestamp: p.timestamp, + source: p.source, + }); + } + } else { + console.error('Failed to fetch FRED WTI crude:', fredOil.error); + errors++; + } + + // FRED: Coal (PCOALAUUSDM — monthly only) + const fredCoal = await fred.getCoalPrice(start, end); + if (fredCoal.ok) { + for (const p of fredCoal.data) { + rows.push({ + commodity: p.commodity, + price: p.price, + unit: p.unit, + timestamp: p.timestamp, + source: p.source, + }); + } + } else { + console.error('Failed to fetch FRED coal:', fredCoal.error); + errors++; + } + + return { rows, errors }; +} + +export async function GET(request: NextRequest): Promise> { + const searchParams = request.nextUrl.searchParams; + const start = searchParams.get('start') ?? undefined; + const end = searchParams.get('end') ?? undefined; + + const stats: IngestionStats = { inserted: 0, updated: 0, errors: 0 }; + + const { rows, errors: fetchErrors } = await fetchAllCommodities(start, end); + stats.errors += fetchErrors; + + if (rows.length === 0) { + return NextResponse.json(stats); + } + + // Deduplicate: for same commodity + timestamp, prefer EIA over FRED + const deduped = new Map(); + for (const row of rows) { + const key = `${row.commodity}:${row.timestamp.getTime()}`; + const existing = deduped.get(key); + if (!existing || (existing.source === 'FRED' && row.source === 'EIA')) { + deduped.set(key, row); + } + } + + const uniqueRows = [...deduped.values()]; + + // Fetch existing records to determine inserts vs updates + const timestamps = uniqueRows.map(r => r.timestamp); + const commodities = [...new Set(uniqueRows.map(r => r.commodity))]; + + const existing = await prisma.commodityPrice.findMany({ + where: { + commodity: { in: commodities }, + timestamp: { in: timestamps }, + }, + select: { id: true, commodity: true, timestamp: true }, + }); + + const existingKeys = new Map(existing.map(e => [`${e.commodity}:${e.timestamp.getTime()}`, e.id])); + + const toCreate: Array<{ + commodity: string; + price: number; + unit: string; + timestamp: Date; + source: string; + }> = []; + const toUpdate: Array<{ id: string; price: number; unit: string; source: string }> = []; + + for (const row of uniqueRows) { + const key = `${row.commodity}:${row.timestamp.getTime()}`; + const existingId = existingKeys.get(key); + if (existingId) { + toUpdate.push({ id: existingId, price: row.price, unit: row.unit, source: row.source }); + } else { + toCreate.push({ + commodity: row.commodity, + price: row.price, + unit: row.unit, + timestamp: row.timestamp, + source: row.source, + }); + } + } + + if (toCreate.length > 0) { + const result = await prisma.commodityPrice.createMany({ data: toCreate }); + stats.inserted += result.count; + } + + if (toUpdate.length > 0) { + await prisma.$transaction( + toUpdate.map(u => + prisma.commodityPrice.update({ + where: { id: u.id }, + data: { price: u.price, unit: u.unit, source: u.source }, + }), + ), + ); + stats.updated += toUpdate.length; + } + + return NextResponse.json(stats); +} diff --git a/src/app/api/ingest/electricity/route.ts b/src/app/api/ingest/electricity/route.ts new file mode 100644 index 0000000..dffb9f6 --- /dev/null +++ b/src/app/api/ingest/electricity/route.ts @@ -0,0 +1,116 @@ +import { NextResponse, type NextRequest } from 'next/server.js'; + +import { getRegionData } from '@/lib/api/eia.js'; +import { prisma } from '@/lib/db.js'; +import { EIA_RESPONDENT_CODES, type RegionCode } from '@/lib/schemas/electricity.js'; + +const ALL_REGIONS: RegionCode[] = ['PJM', 'ERCOT', 'CAISO', 'NYISO', 'ISONE', 'MISO', 'SPP']; + +function isRegionCode(value: string): value is RegionCode { + return value in EIA_RESPONDENT_CODES; +} + +interface IngestionStats { + inserted: number; + updated: number; + errors: number; +} + +export async function GET(request: NextRequest): Promise> { + const searchParams = request.nextUrl.searchParams; + const regionParam = searchParams.get('region'); + const start = searchParams.get('start') ?? undefined; + const end = searchParams.get('end') ?? undefined; + + let regions: RegionCode[]; + if (regionParam) { + if (!isRegionCode(regionParam)) { + return NextResponse.json({ inserted: 0, updated: 0, errors: 1 }, { status: 400 }); + } + regions = [regionParam]; + } else { + regions = ALL_REGIONS; + } + + const stats: IngestionStats = { inserted: 0, updated: 0, errors: 0 }; + + // Pre-fetch all grid regions in one query + const gridRegions = await prisma.gridRegion.findMany({ + where: { code: { in: regions } }, + select: { id: true, code: true }, + }); + const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id])); + + for (const regionCode of regions) { + const regionId = regionIdByCode.get(regionCode); + if (!regionId) { + stats.errors++; + continue; + } + + try { + const demandData = await getRegionData(regionCode, 'D', { start, end }); + const validPoints = demandData.filter((p): p is typeof p & { valueMw: number } => p.valueMw !== null); + + if (validPoints.length === 0) continue; + + // Fetch existing records for this region in the time range to avoid N+1 + const timestamps = validPoints.map(p => p.timestamp); + const existing = await prisma.electricityPrice.findMany({ + where: { + regionId, + timestamp: { in: timestamps }, + }, + select: { id: true, timestamp: true }, + }); + const existingByTime = new Map(existing.map(e => [e.timestamp.getTime(), e.id])); + + const toCreate: Array<{ + regionId: string; + priceMwh: number; + demandMw: number; + timestamp: Date; + source: string; + }> = []; + const toUpdate: Array<{ id: string; demandMw: number }> = []; + + for (const point of validPoints) { + const existingId = existingByTime.get(point.timestamp.getTime()); + if (existingId) { + toUpdate.push({ id: existingId, demandMw: point.valueMw }); + } else { + toCreate.push({ + regionId, + priceMwh: 0, + demandMw: point.valueMw, + timestamp: point.timestamp, + source: 'EIA', + }); + } + } + + if (toCreate.length > 0) { + const result = await prisma.electricityPrice.createMany({ data: toCreate }); + stats.inserted += result.count; + } + + // Batch updates via transaction + if (toUpdate.length > 0) { + await prisma.$transaction( + toUpdate.map(u => + prisma.electricityPrice.update({ + where: { id: u.id }, + data: { demandMw: u.demandMw, source: 'EIA' }, + }), + ), + ); + stats.updated += toUpdate.length; + } + } catch (err) { + console.error(`Failed to ingest electricity data for ${regionCode}:`, err); + stats.errors++; + } + } + + return NextResponse.json(stats); +} diff --git a/src/app/api/ingest/generation/route.ts b/src/app/api/ingest/generation/route.ts new file mode 100644 index 0000000..9f905d4 --- /dev/null +++ b/src/app/api/ingest/generation/route.ts @@ -0,0 +1,114 @@ +import { NextResponse, type NextRequest } from 'next/server.js'; + +import { getFuelTypeData } from '@/lib/api/eia.js'; +import { prisma } from '@/lib/db.js'; +import { EIA_RESPONDENT_CODES, type RegionCode } from '@/lib/schemas/electricity.js'; + +const ALL_REGIONS: RegionCode[] = ['PJM', 'ERCOT', 'CAISO', 'NYISO', 'ISONE', 'MISO', 'SPP']; + +function isRegionCode(value: string): value is RegionCode { + return value in EIA_RESPONDENT_CODES; +} + +interface IngestionStats { + inserted: number; + updated: number; + errors: number; +} + +export async function GET(request: NextRequest): Promise> { + const searchParams = request.nextUrl.searchParams; + const regionParam = searchParams.get('region'); + const start = searchParams.get('start') ?? undefined; + const end = searchParams.get('end') ?? undefined; + + let regions: RegionCode[]; + if (regionParam) { + if (!isRegionCode(regionParam)) { + return NextResponse.json({ inserted: 0, updated: 0, errors: 1 }, { status: 400 }); + } + regions = [regionParam]; + } else { + regions = ALL_REGIONS; + } + + const stats: IngestionStats = { inserted: 0, updated: 0, errors: 0 }; + + // Pre-fetch all grid regions in one query + const gridRegions = await prisma.gridRegion.findMany({ + where: { code: { in: regions } }, + select: { id: true, code: true }, + }); + const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id])); + + for (const regionCode of regions) { + const regionId = regionIdByCode.get(regionCode); + if (!regionId) { + stats.errors++; + continue; + } + + try { + const fuelData = await getFuelTypeData(regionCode, { start, end }); + const validPoints = fuelData.filter((p): p is typeof p & { generationMw: number } => p.generationMw !== null); + + if (validPoints.length === 0) continue; + + // Fetch existing records for this region to determine inserts vs updates + const timestamps = validPoints.map(p => p.timestamp); + const existing = await prisma.generationMix.findMany({ + where: { + regionId, + timestamp: { in: timestamps }, + }, + select: { id: true, timestamp: true, fuelType: true }, + }); + const existingKeys = new Map(existing.map(e => [`${e.fuelType}:${e.timestamp.getTime()}`, e.id])); + + const toCreate: Array<{ + regionId: string; + fuelType: string; + generationMw: number; + timestamp: Date; + }> = []; + const toUpdate: Array<{ id: string; generationMw: number }> = []; + + for (const point of validPoints) { + const key = `${point.fuelType}:${point.timestamp.getTime()}`; + const existingId = existingKeys.get(key); + if (existingId) { + toUpdate.push({ id: existingId, generationMw: point.generationMw }); + } else { + toCreate.push({ + regionId, + fuelType: point.fuelType, + generationMw: point.generationMw, + timestamp: point.timestamp, + }); + } + } + + if (toCreate.length > 0) { + const result = await prisma.generationMix.createMany({ data: toCreate }); + stats.inserted += result.count; + } + + if (toUpdate.length > 0) { + await prisma.$transaction( + toUpdate.map(u => + prisma.generationMix.update({ + where: { id: u.id }, + data: { generationMw: u.generationMw }, + }), + ), + ); + stats.updated += toUpdate.length; + } + } catch (err) { + console.error(`Failed to ingest generation data for ${regionCode}:`, err); + stats.errors++; + } + } + + return NextResponse.json(stats); +} diff --git a/src/lib/api/eia.ts b/src/lib/api/eia.ts new file mode 100644 index 0000000..034bad9 --- /dev/null +++ b/src/lib/api/eia.ts @@ -0,0 +1,339 @@ +import { + type CommodityPricePoint, + eiaNaturalGasResponseSchema, + eiaWtiCrudeResponseSchema, + parseEiaCommodityPeriod, +} from '@/lib/schemas/commodities.js'; +import { + EIA_RESPONDENT_CODES, + type EiaFuelTypeDataRow, + type EiaRegionDataRow, + type FuelTypeDataPoint, + type RegionCode, + type RegionDataPoint, + eiaFuelTypeDataResponseSchema, + eiaRegionDataResponseSchema, + parseEiaPeriod, + resolveRegionCode, +} from '@/lib/schemas/electricity.js'; + +const EIA_BASE_URL = 'https://api.eia.gov/v2'; +const MAX_ROWS_PER_REQUEST = 5000; + +function getApiKey(): string { + const key = process.env.EIA_API_KEY; + if (!key) { + throw new Error('EIA_API_KEY environment variable is not set'); + } + return key; +} + +interface EiaQueryParams { + frequency?: 'hourly' | 'daily' | 'weekly' | 'monthly' | 'annual'; + start?: string; + end?: string; + facets?: Record; + sort?: Array<{ column: string; direction: 'asc' | 'desc' }>; + offset?: number; + length?: number; +} + +function buildUrl(endpoint: string, params: EiaQueryParams): string { + const url = new URL(`${EIA_BASE_URL}${endpoint}`); + url.searchParams.set('api_key', getApiKey()); + url.searchParams.set('data[0]', 'value'); + + if (params.frequency) { + url.searchParams.set('frequency', params.frequency); + } + if (params.start) { + url.searchParams.set('start', params.start); + } + if (params.end) { + url.searchParams.set('end', params.end); + } + if (params.facets) { + for (const [key, values] of Object.entries(params.facets)) { + for (const value of values) { + url.searchParams.append(`facets[${key}][]`, value); + } + } + } + if (params.sort) { + for (let i = 0; i < params.sort.length; i++) { + const s = params.sort[i]; + if (s) { + url.searchParams.set(`sort[${i}][column]`, s.column); + url.searchParams.set(`sort[${i}][direction]`, s.direction); + } + } + } + + url.searchParams.set('offset', String(params.offset ?? 0)); + url.searchParams.set('length', String(params.length ?? MAX_ROWS_PER_REQUEST)); + + return url.toString(); +} + +async function fetchEia(endpoint: string, params: EiaQueryParams): Promise { + const url = buildUrl(endpoint, params); + const response = await fetch(url); + + if (!response.ok) { + const text = await response.text().catch(() => 'unknown error'); + throw new Error(`EIA API error ${response.status}: ${text}`); + } + + return response.json(); +} + +/** + * Auto-paginate through an EIA endpoint, collecting all rows. + * EIA limits responses to 5000 rows — this fetches all pages sequentially. + */ +async function fetchAllPages( + endpoint: string, + params: EiaQueryParams, + parseResponse: (json: unknown) => { total: number; data: T[] }, +): Promise { + const allData: T[] = []; + let offset = params.offset ?? 0; + const length = params.length ?? MAX_ROWS_PER_REQUEST; + + for (;;) { + const json = await fetchEia(endpoint, { ...params, offset, length }); + const result = parseResponse(json); + + allData.push(...result.data); + + if (allData.length >= result.total || result.data.length < length) { + break; + } + + offset += length; + } + + return allData; +} + +/** Convert a raw EIA region-data row into a typed RegionDataPoint */ +function transformRegionDataRow(row: EiaRegionDataRow): RegionDataPoint { + const regionCode = resolveRegionCode(row.respondent); + + return { + timestamp: parseEiaPeriod(row.period), + regionCode: regionCode ?? 'PJM', + respondent: row.respondent, + type: row.type, + typeName: row['type-name'], + valueMw: row.value, + valueUnits: row['value-units'], + }; +} + +/** Convert a raw EIA fuel-type-data row into a typed FuelTypeDataPoint */ +function transformFuelTypeRow(row: EiaFuelTypeDataRow): FuelTypeDataPoint { + const regionCode = resolveRegionCode(row.respondent); + + return { + timestamp: parseEiaPeriod(row.period), + regionCode: regionCode ?? 'PJM', + respondent: row.respondent, + fuelType: row.fueltype, + typeName: row['type-name'], + generationMw: row.value, + valueUnits: row['value-units'], + }; +} + +export interface GetRegionDataOptions { + start?: string; + end?: string; + /** Maximum number of rows to fetch. Omit for all available data. */ + limit?: number; +} + +/** + * Fetch hourly demand or net generation data for a region. + * + * @param regionCode - One of PJM, ERCOT, CAISO, NYISO, ISONE, MISO, SPP + * @param type - "D" for demand, "NG" for net generation + * + * NOTE: EIA does not provide real-time wholesale electricity prices. + * The /v2/electricity/rto/region-data/data/ endpoint provides demand (MW) and + * net generation (MW), NOT price ($/MWh). For the electricity_prices table, + * we store demand_mw from this data; price_mwh will be null. + * TODO: Investigate scraping ISO-specific price feeds (PJM LMP, ERCOT SPP, CAISO LMP) + * for real wholesale price data if needed in the future. + */ +export async function getRegionData( + regionCode: RegionCode, + type: 'D' | 'NG', + options: GetRegionDataOptions = {}, +): Promise { + const respondentCode = EIA_RESPONDENT_CODES[regionCode]; + + const params: EiaQueryParams = { + frequency: 'hourly', + start: options.start, + end: options.end, + facets: { + respondent: [respondentCode], + type: [type], + }, + sort: [{ column: 'period', direction: 'desc' }], + length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST, + }; + + if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) { + const json = await fetchEia('/electricity/rto/region-data/data/', params); + const parsed = eiaRegionDataResponseSchema.parse(json); + return parsed.response.data.map(transformRegionDataRow); + } + + const rows = await fetchAllPages('/electricity/rto/region-data/data/', params, json => { + const parsed = eiaRegionDataResponseSchema.parse(json); + return { total: parsed.response.total, data: parsed.response.data }; + }); + + return rows.map(transformRegionDataRow); +} + +export interface GetFuelTypeDataOptions { + start?: string; + end?: string; + fuelType?: string; + limit?: number; +} + +/** + * Fetch hourly generation by fuel type for a region. + * + * @param regionCode - One of PJM, ERCOT, CAISO, NYISO, ISONE, MISO, SPP + * @param options - Optional filters for time range and fuel type + */ +export async function getFuelTypeData( + regionCode: RegionCode, + options: GetFuelTypeDataOptions = {}, +): Promise { + const respondentCode = EIA_RESPONDENT_CODES[regionCode]; + + const facets: Record = { + respondent: [respondentCode], + }; + if (options.fuelType) { + facets['fueltype'] = [options.fuelType]; + } + + const params: EiaQueryParams = { + frequency: 'hourly', + start: options.start, + end: options.end, + facets, + sort: [{ column: 'period', direction: 'desc' }], + length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST, + }; + + if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) { + const json = await fetchEia('/electricity/rto/fuel-type-data/data/', params); + const parsed = eiaFuelTypeDataResponseSchema.parse(json); + return parsed.response.data.map(transformFuelTypeRow); + } + + const rows = await fetchAllPages('/electricity/rto/fuel-type-data/data/', params, json => { + const parsed = eiaFuelTypeDataResponseSchema.parse(json); + return { total: parsed.response.total, data: parsed.response.data }; + }); + + return rows.map(transformFuelTypeRow); +} + +export interface GetCommodityPriceOptions { + start?: string; + end?: string; + limit?: number; +} + +/** + * Fetch Henry Hub natural gas spot prices. + * Endpoint: /v2/natural-gas/pri/fut/data/ with facets[series][]=RNGWHHD + */ +export async function getNaturalGasPrice(options: GetCommodityPriceOptions = {}): Promise { + const params: EiaQueryParams = { + frequency: 'daily', + start: options.start, + end: options.end, + facets: { + series: ['RNGWHHD'], + }, + sort: [{ column: 'period', direction: 'desc' }], + length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST, + }; + + if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) { + const json = await fetchEia('/natural-gas/pri/fut/data/', params); + const parsed = eiaNaturalGasResponseSchema.parse(json); + return parsed.response.data.map(row => ({ + timestamp: parseEiaCommodityPeriod(row.period), + commodity: 'natural_gas' as const, + price: row.value, + unit: row.units ?? '$/Million BTU', + source: 'EIA', + })); + } + + const rows = await fetchAllPages('/natural-gas/pri/fut/data/', params, json => { + const parsed = eiaNaturalGasResponseSchema.parse(json); + return { total: parsed.response.total, data: parsed.response.data }; + }); + + return rows.map(row => ({ + timestamp: parseEiaCommodityPeriod(row.period), + commodity: 'natural_gas' as const, + price: row.value, + unit: row.units ?? '$/Million BTU', + source: 'EIA', + })); +} + +/** + * Fetch WTI crude oil spot prices. + * Endpoint: /v2/petroleum/pri/spt/data/ with facets[series][]=RWTC + */ +export async function getWTICrudePrice(options: GetCommodityPriceOptions = {}): Promise { + const params: EiaQueryParams = { + frequency: 'daily', + start: options.start, + end: options.end, + facets: { + series: ['RWTC'], + }, + sort: [{ column: 'period', direction: 'desc' }], + length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST, + }; + + if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) { + const json = await fetchEia('/petroleum/pri/spt/data/', params); + const parsed = eiaWtiCrudeResponseSchema.parse(json); + return parsed.response.data.map(row => ({ + timestamp: parseEiaCommodityPeriod(row.period), + commodity: 'wti_crude' as const, + price: row.value, + unit: row.units ?? '$/Barrel', + source: 'EIA', + })); + } + + const rows = await fetchAllPages('/petroleum/pri/spt/data/', params, json => { + const parsed = eiaWtiCrudeResponseSchema.parse(json); + return { total: parsed.response.total, data: parsed.response.data }; + }); + + return rows.map(row => ({ + timestamp: parseEiaCommodityPeriod(row.period), + commodity: 'wti_crude' as const, + price: row.value, + unit: row.units ?? '$/Barrel', + source: 'EIA', + })); +} diff --git a/src/lib/api/fred.ts b/src/lib/api/fred.ts new file mode 100644 index 0000000..78ae067 --- /dev/null +++ b/src/lib/api/fred.ts @@ -0,0 +1,248 @@ +import { z } from 'zod'; + +import { COMMODITY_UNITS, type CommodityPrice, type CommodityType } from '@/lib/schemas/commodities.js'; + +// --------------------------------------------------------------------------- +// FRED Series IDs +// --------------------------------------------------------------------------- + +const FRED_SERIES = { + natural_gas: 'DHHNGSP', + wti_crude: 'DCOILWTICO', + coal: 'PCOALAUUSDM', +} as const satisfies Record; + +// --------------------------------------------------------------------------- +// Zod Schemas — FRED API Response +// --------------------------------------------------------------------------- + +const FredObservationSchema = z.object({ + realtime_start: z.string(), + realtime_end: z.string(), + date: z.string(), // "YYYY-MM-DD" + value: z.string(), // numeric string or "." for missing +}); + +const FredSeriesResponseSchema = z.object({ + realtime_start: z.string(), + realtime_end: z.string(), + observation_start: z.string(), + observation_end: z.string(), + units: z.string(), + output_type: z.number(), + file_type: z.string(), + order_by: z.string(), + sort_order: z.string(), + count: z.number(), + offset: z.number(), + limit: z.number(), + observations: z.array(FredObservationSchema), +}); + +export type FredObservation = z.infer; +export type FredSeriesResponse = z.infer; + +// --------------------------------------------------------------------------- +// Parsed observation (after filtering missing "." values) +// --------------------------------------------------------------------------- + +export interface FredDataPoint { + date: Date; // UTC midnight for the observation date + value: number; +} + +// --------------------------------------------------------------------------- +// Error type +// --------------------------------------------------------------------------- + +export interface FredApiError { + ok: false; + error: string; + status?: number; +} + +export interface FredApiSuccess { + ok: true; + data: T; +} + +export type FredApiResult = FredApiSuccess | FredApiError; + +// --------------------------------------------------------------------------- +// Rate limiter — 50ms delay between requests +// --------------------------------------------------------------------------- + +let lastRequestTime = 0; + +async function rateLimitDelay(): Promise { + const now = Date.now(); + const elapsed = now - lastRequestTime; + if (elapsed < 50) { + await new Promise(resolve => setTimeout(resolve, 50 - elapsed)); + } + lastRequestTime = Date.now(); +} + +// --------------------------------------------------------------------------- +// Core fetcher +// --------------------------------------------------------------------------- + +const FRED_BASE_URL = 'https://api.stlouisfed.org/fred/series/observations'; + +export interface GetSeriesOptions { + observationStart?: string; // "YYYY-MM-DD" + observationEnd?: string; // "YYYY-MM-DD" + limit?: number; // default 100000 (FRED max) + offset?: number; +} + +export async function getSeriesObservations( + seriesId: string, + options: GetSeriesOptions = {}, +): Promise> { + const apiKey = process.env.FRED_API_KEY; + if (!apiKey) { + return { ok: false, error: 'FRED_API_KEY is not set' }; + } + + const params = new URLSearchParams({ + series_id: seriesId, + api_key: apiKey, + file_type: 'json', + }); + + if (options.observationStart) { + params.set('observation_start', options.observationStart); + } + if (options.observationEnd) { + params.set('observation_end', options.observationEnd); + } + if (options.limit !== undefined) { + params.set('limit', String(options.limit)); + } + if (options.offset !== undefined) { + params.set('offset', String(options.offset)); + } + + await rateLimitDelay(); + + let response: Response; + try { + response = await fetch(`${FRED_BASE_URL}?${params.toString()}`); + } catch (err) { + return { + ok: false, + error: `FRED API network error: ${err instanceof Error ? err.message : String(err)}`, + }; + } + + if (!response.ok) { + const body = await response.text().catch(() => ''); + return { + ok: false, + error: `FRED API returned ${response.status}: ${body}`, + status: response.status, + }; + } + + let json: unknown; + try { + json = await response.json(); + } catch { + return { ok: false, error: 'FRED API returned invalid JSON' }; + } + + const parsed = FredSeriesResponseSchema.safeParse(json); + if (!parsed.success) { + return { + ok: false, + error: `FRED response validation failed: ${parsed.error.message}`, + }; + } + + const dataPoints = parseObservations(parsed.data.observations); + + return { ok: true, data: dataPoints }; +} + +// --------------------------------------------------------------------------- +// Parse observations — filter out missing values (".") +// --------------------------------------------------------------------------- + +function parseObservations(observations: FredObservation[]): FredDataPoint[] { + const points: FredDataPoint[] = []; + + for (const obs of observations) { + if (obs.value === '.') continue; + + const numValue = Number(obs.value); + if (Number.isNaN(numValue)) continue; + + points.push({ + date: new Date(`${obs.date}T00:00:00Z`), + value: numValue, + }); + } + + return points; +} + +// --------------------------------------------------------------------------- +// Convenience wrappers +// --------------------------------------------------------------------------- + +function formatDateParam(date?: Date | string): string | undefined { + if (!date) return undefined; + if (typeof date === 'string') return date; + return date.toISOString().slice(0, 10); +} + +export async function getNaturalGasPrice( + startDate?: Date | string, + endDate?: Date | string, +): Promise> { + return getCommodityPrices('natural_gas', startDate, endDate); +} + +export async function getWTICrudePrice( + startDate?: Date | string, + endDate?: Date | string, +): Promise> { + return getCommodityPrices('wti_crude', startDate, endDate); +} + +export async function getCoalPrice( + startDate?: Date | string, + endDate?: Date | string, +): Promise> { + return getCommodityPrices('coal', startDate, endDate); +} + +// --------------------------------------------------------------------------- +// Shared commodity fetcher +// --------------------------------------------------------------------------- + +async function getCommodityPrices( + commodity: CommodityType, + startDate?: Date | string, + endDate?: Date | string, +): Promise> { + const seriesId = FRED_SERIES[commodity]; + const result = await getSeriesObservations(seriesId, { + observationStart: formatDateParam(startDate), + observationEnd: formatDateParam(endDate), + }); + + if (!result.ok) return result; + + const unit = COMMODITY_UNITS[commodity]; + const prices: CommodityPrice[] = result.data.map(point => ({ + commodity, + price: point.value, + unit, + timestamp: point.date, + source: 'FRED', + })); + + return { ok: true, data: prices }; +} diff --git a/src/lib/schemas/commodities.ts b/src/lib/schemas/commodities.ts new file mode 100644 index 0000000..a5372b8 --- /dev/null +++ b/src/lib/schemas/commodities.ts @@ -0,0 +1,94 @@ +import { z } from 'zod'; + +export const CommodityType = z.enum(['natural_gas', 'wti_crude', 'coal']); +export type CommodityType = z.infer; + +export const CommodityPriceSchema = z.object({ + commodity: CommodityType, + price: z.number(), + unit: z.string(), + timestamp: z.date(), + source: z.string(), +}); +export type CommodityPrice = z.infer; + +export const COMMODITY_UNITS: Record = { + natural_gas: '$/Million BTU', + wti_crude: '$/Barrel', + coal: '$/Metric Ton', +}; + +/** Coerce EIA string values to numbers, treating empty/null as null */ +const eiaNumericValue = z.union([z.string(), z.number(), z.null()]).transform((val): number | null => { + if (val === null || val === '') return null; + const num = Number(val); + return Number.isNaN(num) ? null : num; +}); + +/** + * EIA natural gas price row. + * Endpoint: /v2/natural-gas/pri/fut/data/ + * Series: RNGWHHD (Henry Hub Natural Gas Spot Price) + */ +export const eiaNaturalGasRowSchema = z.object({ + period: z.string(), + 'series-description': z.string(), + value: eiaNumericValue, + units: z.string().optional(), +}); + +export type EiaNaturalGasRow = z.infer; + +/** + * EIA petroleum/crude oil price row. + * Endpoint: /v2/petroleum/pri/spt/data/ + * Series: RWTC (WTI Cushing Oklahoma Spot Price) + */ +export const eiaWtiCrudeRowSchema = z.object({ + period: z.string(), + 'series-description': z.string().optional(), + value: eiaNumericValue, + units: z.string().optional(), +}); + +export type EiaWtiCrudeRow = z.infer; + +/** + * Parsed commodity price data point (from EIA). + */ +export interface CommodityPricePoint { + timestamp: Date; + commodity: 'natural_gas' | 'wti_crude'; + price: number | null; + unit: string; + source: string; +} + +/** + * Generic EIA API response wrapper for commodity endpoints. + */ +export function eiaCommodityResponseSchema(dataRowSchema: T) { + return z.object({ + response: z.object({ + total: z.union([z.string(), z.number()]).transform(Number), + data: z.array(dataRowSchema), + }), + }); +} + +export const eiaNaturalGasResponseSchema = eiaCommodityResponseSchema(eiaNaturalGasRowSchema); +export const eiaWtiCrudeResponseSchema = eiaCommodityResponseSchema(eiaWtiCrudeRowSchema); + +export type EiaNaturalGasResponse = z.infer; +export type EiaWtiCrudeResponse = z.infer; + +/** + * Parse an EIA daily period string to a UTC Date. + * Commodity data uses daily format: "2026-02-02" + */ +export function parseEiaCommodityPeriod(period: string): Date { + if (/^\d{4}-\d{2}-\d{2}$/.test(period)) { + return new Date(`${period}T00:00:00Z`); + } + return new Date(period); +} diff --git a/src/lib/schemas/electricity.ts b/src/lib/schemas/electricity.ts new file mode 100644 index 0000000..3a4c9e6 --- /dev/null +++ b/src/lib/schemas/electricity.ts @@ -0,0 +1,152 @@ +import { z } from 'zod'; + +/** + * EIA respondent codes for the 7 ISOs we track. + * Note: EIA uses different codes than the common abbreviations. + */ +export const EIA_RESPONDENT_CODES = { + PJM: 'PJM', + ERCOT: 'ERCO', + CAISO: 'CISO', + NYISO: 'NYIS', + ISONE: 'ISNE', + MISO: 'MISO', + SPP: 'SWPP', +} as const; + +export type RegionCode = keyof typeof EIA_RESPONDENT_CODES; +export type EiaRespondentCode = (typeof EIA_RESPONDENT_CODES)[RegionCode]; + +/** Maps EIA respondent codes back to our region codes */ +export const RESPONDENT_TO_REGION: Record = { + PJM: 'PJM', + ERCO: 'ERCOT', + CISO: 'CAISO', + NYIS: 'NYISO', + ISNE: 'ISONE', + MISO: 'MISO', + SWPP: 'SPP', +}; + +/** Type guard: check if a string is a valid EIA respondent code */ +export function isEiaRespondentCode(code: string): code is EiaRespondentCode { + return code in RESPONDENT_TO_REGION; +} + +/** Safely resolve a respondent string to a RegionCode, or undefined if unknown */ +export function resolveRegionCode(respondent: string): RegionCode | undefined { + if (isEiaRespondentCode(respondent)) { + return RESPONDENT_TO_REGION[respondent]; + } + return undefined; +} + +/** Coerce EIA string values to numbers, treating empty/null as null */ +const eiaNumericValue = z.union([z.string(), z.number(), z.null()]).transform((val): number | null => { + if (val === null || val === '') return null; + const num = Number(val); + return Number.isNaN(num) ? null : num; +}); + +/** + * EIA region-data response row. + * Endpoint: /v2/electricity/rto/region-data/data/ + * Provides hourly demand (D) and net generation (NG) by balancing authority. + */ +export const eiaRegionDataRowSchema = z.object({ + period: z.string(), + respondent: z.string(), + 'respondent-name': z.string(), + type: z.string(), + 'type-name': z.string(), + value: eiaNumericValue, + 'value-units': z.string(), +}); + +export type EiaRegionDataRow = z.infer; + +/** + * EIA fuel-type-data response row. + * Endpoint: /v2/electricity/rto/fuel-type-data/data/ + * Provides hourly generation by fuel type (e.g., NG, SUN, WND, NUC, WAT, COL). + */ +export const eiaFuelTypeDataRowSchema = z.object({ + period: z.string(), + respondent: z.string(), + 'respondent-name': z.string(), + fueltype: z.string(), + 'type-name': z.string(), + value: eiaNumericValue, + 'value-units': z.string(), +}); + +export type EiaFuelTypeDataRow = z.infer; + +/** + * Parsed region data after transformation from the raw EIA response. + */ +export interface RegionDataPoint { + timestamp: Date; + regionCode: RegionCode; + respondent: string; + type: string; + typeName: string; + valueMw: number | null; + valueUnits: string; +} + +/** + * Parsed fuel type data after transformation. + */ +export interface FuelTypeDataPoint { + timestamp: Date; + regionCode: RegionCode; + respondent: string; + fuelType: string; + typeName: string; + generationMw: number | null; + valueUnits: string; +} + +/** + * EIA API pagination metadata. + * Note: `total` comes back as a string from EIA. + */ +export const eiaPaginationSchema = z.object({ + offset: z.number(), + length: z.number(), + total: z.union([z.string(), z.number()]).transform(Number), +}); + +/** + * Generic EIA API response wrapper. + * All EIA v2 responses share this structure. + */ +export function eiaResponseSchema(dataRowSchema: T) { + return z.object({ + response: z.object({ + total: z.union([z.string(), z.number()]).transform(Number), + data: z.array(dataRowSchema), + }), + }); +} + +export const eiaRegionDataResponseSchema = eiaResponseSchema(eiaRegionDataRowSchema); +export const eiaFuelTypeDataResponseSchema = eiaResponseSchema(eiaFuelTypeDataRowSchema); + +export type EiaRegionDataResponse = z.infer; +export type EiaFuelTypeDataResponse = z.infer; + +/** + * Parse an EIA hourly period string to a UTC Date. + * Format: "2026-02-11T08" => Date for 2026-02-11 08:00:00 UTC + */ +export function parseEiaPeriod(period: string): Date { + if (/^\d{4}-\d{2}-\d{2}T\d{2}$/.test(period)) { + return new Date(`${period}:00:00Z`); + } + if (/^\d{4}-\d{2}-\d{2}$/.test(period)) { + return new Date(`${period}T00:00:00Z`); + } + return new Date(period); +} diff --git a/tsconfig.json b/tsconfig.json index ad34ae1..053eb72 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -43,7 +43,11 @@ "paths": { "@/*": [ "./src/*" - ] + ], + "next/font": ["./node_modules/next/font/index.d.ts"], + "next/font/local": ["./node_modules/next/font/local/index.d.ts"], + "next/font/google": ["./node_modules/next/font/google/index.d.ts"], + "next/*": ["./node_modules/next/*.d.ts"] } }, "include": [ @@ -51,6 +55,7 @@ "next.config.ts", "prisma.config.ts", "prisma/seed.ts", + "scripts/**/*.ts", "src/**/*.ts", "src/**/*.tsx", ".next/types/**/*.ts",