phase 2: data layer — EIA/FRED clients, server actions, ingestion routes, backfill
This commit is contained in:
parent
6d7d2d966b
commit
64f7099603
424
scripts/backfill.ts
Normal file
424
scripts/backfill.ts
Normal file
@ -0,0 +1,424 @@
|
||||
/**
|
||||
* Historical data backfill script.
|
||||
*
|
||||
* Populates 6 months of historical data from EIA and FRED into Postgres.
|
||||
* Idempotent — safe to re-run; existing records are updated, not duplicated.
|
||||
*
|
||||
* Usage: bun run scripts/backfill.ts
|
||||
*/
|
||||
|
||||
import 'dotenv/config';
|
||||
|
||||
import { PrismaPg } from '@prisma/adapter-pg';
|
||||
import { PrismaClient } from '../src/generated/prisma/client.js';
|
||||
|
||||
import * as eia from '../src/lib/api/eia.js';
|
||||
import { getFuelTypeData, getRegionData } from '../src/lib/api/eia.js';
|
||||
import * as fred from '../src/lib/api/fred.js';
|
||||
import type { RegionCode } from '../src/lib/schemas/electricity.js';
|
||||
|
||||
const adapter = new PrismaPg({ connectionString: process.env.DATABASE_URL });
|
||||
const prisma = new PrismaClient({ adapter });
|
||||
|
||||
const ALL_REGIONS: RegionCode[] = ['PJM', 'ERCOT', 'CAISO', 'NYISO', 'ISONE', 'MISO', 'SPP'];
|
||||
|
||||
const SIX_MONTHS_MS = 6 * 30 * 24 * 60 * 60 * 1000;
|
||||
|
||||
function sixMonthsAgoIso(): string {
|
||||
return new Date(Date.now() - SIX_MONTHS_MS).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function todayIso(): string {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function log(msg: string): void {
|
||||
const ts = new Date().toISOString().slice(11, 19);
|
||||
console.log(`[${ts}] ${msg}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Electricity demand backfill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function backfillElectricity(): Promise<void> {
|
||||
log('=== Backfilling electricity demand data ===');
|
||||
|
||||
const gridRegions = await prisma.gridRegion.findMany({
|
||||
select: { id: true, code: true },
|
||||
});
|
||||
const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id]));
|
||||
|
||||
const start = sixMonthsAgoIso();
|
||||
const end = todayIso();
|
||||
|
||||
for (const regionCode of ALL_REGIONS) {
|
||||
const regionId = regionIdByCode.get(regionCode);
|
||||
if (!regionId) {
|
||||
log(` SKIP ${regionCode} — no grid_region row found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
log(` Fetching demand for ${regionCode}...`);
|
||||
try {
|
||||
const demandData = await getRegionData(regionCode, 'D', { start, end });
|
||||
const validPoints = demandData.filter((p): p is typeof p & { valueMw: number } => p.valueMw !== null);
|
||||
|
||||
if (validPoints.length === 0) {
|
||||
log(` ${regionCode}: 0 valid data points`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check existing records to decide create vs update
|
||||
const timestamps = validPoints.map(p => p.timestamp);
|
||||
const existing = await prisma.electricityPrice.findMany({
|
||||
where: { regionId, timestamp: { in: timestamps } },
|
||||
select: { id: true, timestamp: true },
|
||||
});
|
||||
const existingByTime = new Map(existing.map(e => [e.timestamp.getTime(), e.id]));
|
||||
|
||||
const toCreate: Array<{
|
||||
regionId: string;
|
||||
priceMwh: number;
|
||||
demandMw: number;
|
||||
timestamp: Date;
|
||||
source: string;
|
||||
}> = [];
|
||||
const toUpdate: Array<{ id: string; demandMw: number }> = [];
|
||||
|
||||
for (const point of validPoints) {
|
||||
const existingId = existingByTime.get(point.timestamp.getTime());
|
||||
if (existingId) {
|
||||
toUpdate.push({ id: existingId, demandMw: point.valueMw });
|
||||
} else {
|
||||
toCreate.push({
|
||||
regionId,
|
||||
priceMwh: 0, // TODO: No real-time wholesale price available from EIA
|
||||
demandMw: point.valueMw,
|
||||
timestamp: point.timestamp,
|
||||
source: 'EIA',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (toCreate.length > 0) {
|
||||
const result = await prisma.electricityPrice.createMany({ data: toCreate });
|
||||
log(` ${regionCode}: ${result.count} records inserted`);
|
||||
}
|
||||
|
||||
if (toUpdate.length > 0) {
|
||||
// Batch updates in chunks of 100 to avoid transaction timeouts
|
||||
const chunkSize = 100;
|
||||
for (let i = 0; i < toUpdate.length; i += chunkSize) {
|
||||
const chunk = toUpdate.slice(i, i + chunkSize);
|
||||
await prisma.$transaction(
|
||||
chunk.map(u =>
|
||||
prisma.electricityPrice.update({
|
||||
where: { id: u.id },
|
||||
data: { demandMw: u.demandMw, source: 'EIA' },
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
log(` ${regionCode}: ${toUpdate.length} records updated`);
|
||||
}
|
||||
|
||||
if (toCreate.length === 0 && toUpdate.length === 0) {
|
||||
log(` ${regionCode}: no changes needed`);
|
||||
}
|
||||
} catch (err) {
|
||||
log(` ERROR ${regionCode}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
// Rate limit: 200ms between regions
|
||||
await sleep(200);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generation mix backfill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function backfillGeneration(): Promise<void> {
|
||||
log('=== Backfilling generation mix data ===');
|
||||
|
||||
const gridRegions = await prisma.gridRegion.findMany({
|
||||
select: { id: true, code: true },
|
||||
});
|
||||
const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id]));
|
||||
|
||||
const start = sixMonthsAgoIso();
|
||||
const end = todayIso();
|
||||
|
||||
for (const regionCode of ALL_REGIONS) {
|
||||
const regionId = regionIdByCode.get(regionCode);
|
||||
if (!regionId) {
|
||||
log(` SKIP ${regionCode} — no grid_region row found`);
|
||||
continue;
|
||||
}
|
||||
|
||||
log(` Fetching generation mix for ${regionCode}...`);
|
||||
try {
|
||||
const fuelData = await getFuelTypeData(regionCode, { start, end });
|
||||
const validPoints = fuelData.filter((p): p is typeof p & { generationMw: number } => p.generationMw !== null);
|
||||
|
||||
if (validPoints.length === 0) {
|
||||
log(` ${regionCode}: 0 valid data points`);
|
||||
continue;
|
||||
}
|
||||
|
||||
const timestamps = validPoints.map(p => p.timestamp);
|
||||
const existing = await prisma.generationMix.findMany({
|
||||
where: { regionId, timestamp: { in: timestamps } },
|
||||
select: { id: true, timestamp: true, fuelType: true },
|
||||
});
|
||||
const existingKeys = new Map(existing.map(e => [`${e.fuelType}:${e.timestamp.getTime()}`, e.id]));
|
||||
|
||||
const toCreate: Array<{
|
||||
regionId: string;
|
||||
fuelType: string;
|
||||
generationMw: number;
|
||||
timestamp: Date;
|
||||
}> = [];
|
||||
const toUpdate: Array<{ id: string; generationMw: number }> = [];
|
||||
|
||||
for (const point of validPoints) {
|
||||
const key = `${point.fuelType}:${point.timestamp.getTime()}`;
|
||||
const existingId = existingKeys.get(key);
|
||||
if (existingId) {
|
||||
toUpdate.push({ id: existingId, generationMw: point.generationMw });
|
||||
} else {
|
||||
toCreate.push({
|
||||
regionId,
|
||||
fuelType: point.fuelType,
|
||||
generationMw: point.generationMw,
|
||||
timestamp: point.timestamp,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (toCreate.length > 0) {
|
||||
const result = await prisma.generationMix.createMany({ data: toCreate });
|
||||
log(` ${regionCode}: ${result.count} generation records inserted`);
|
||||
}
|
||||
|
||||
if (toUpdate.length > 0) {
|
||||
const chunkSize = 100;
|
||||
for (let i = 0; i < toUpdate.length; i += chunkSize) {
|
||||
const chunk = toUpdate.slice(i, i + chunkSize);
|
||||
await prisma.$transaction(
|
||||
chunk.map(u =>
|
||||
prisma.generationMix.update({
|
||||
where: { id: u.id },
|
||||
data: { generationMw: u.generationMw },
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
log(` ${regionCode}: ${toUpdate.length} generation records updated`);
|
||||
}
|
||||
|
||||
if (toCreate.length === 0 && toUpdate.length === 0) {
|
||||
log(` ${regionCode}: no changes needed`);
|
||||
}
|
||||
} catch (err) {
|
||||
log(` ERROR ${regionCode}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
await sleep(200);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Commodity prices backfill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface CommodityRow {
|
||||
commodity: string;
|
||||
price: number;
|
||||
unit: string;
|
||||
timestamp: Date;
|
||||
source: string;
|
||||
}
|
||||
|
||||
async function backfillCommodities(): Promise<void> {
|
||||
log('=== Backfilling commodity prices ===');
|
||||
|
||||
const start = sixMonthsAgoIso();
|
||||
const end = todayIso();
|
||||
const rows: CommodityRow[] = [];
|
||||
|
||||
// EIA: Natural Gas (Henry Hub)
|
||||
log(' Fetching EIA natural gas prices...');
|
||||
try {
|
||||
const gasData = await eia.getNaturalGasPrice({ start, end });
|
||||
for (const p of gasData) {
|
||||
if (p.price === null) continue;
|
||||
rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source });
|
||||
}
|
||||
log(` EIA natural gas: ${gasData.length} raw records`);
|
||||
} catch (err) {
|
||||
log(` ERROR EIA natural gas: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
await sleep(200);
|
||||
|
||||
// EIA: WTI Crude
|
||||
log(' Fetching EIA WTI crude prices...');
|
||||
try {
|
||||
const oilData = await eia.getWTICrudePrice({ start, end });
|
||||
for (const p of oilData) {
|
||||
if (p.price === null) continue;
|
||||
rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source });
|
||||
}
|
||||
log(` EIA WTI crude: ${oilData.length} raw records`);
|
||||
} catch (err) {
|
||||
log(` ERROR EIA WTI crude: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
await sleep(200);
|
||||
|
||||
// FRED: Natural Gas (DHHNGSP)
|
||||
log(' Fetching FRED natural gas prices...');
|
||||
const fredGas = await fred.getNaturalGasPrice(start, end);
|
||||
if (fredGas.ok) {
|
||||
for (const p of fredGas.data) {
|
||||
rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source });
|
||||
}
|
||||
log(` FRED natural gas: ${fredGas.data.length} records`);
|
||||
} else {
|
||||
log(` ERROR FRED natural gas: ${fredGas.error}`);
|
||||
}
|
||||
|
||||
await sleep(200);
|
||||
|
||||
// FRED: WTI Crude (DCOILWTICO)
|
||||
log(' Fetching FRED WTI crude prices...');
|
||||
const fredOil = await fred.getWTICrudePrice(start, end);
|
||||
if (fredOil.ok) {
|
||||
for (const p of fredOil.data) {
|
||||
rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source });
|
||||
}
|
||||
log(` FRED WTI crude: ${fredOil.data.length} records`);
|
||||
} else {
|
||||
log(` ERROR FRED WTI crude: ${fredOil.error}`);
|
||||
}
|
||||
|
||||
await sleep(200);
|
||||
|
||||
// FRED: Coal (PCOALAUUSDM)
|
||||
log(' Fetching FRED coal prices...');
|
||||
const fredCoal = await fred.getCoalPrice(start, end);
|
||||
if (fredCoal.ok) {
|
||||
for (const p of fredCoal.data) {
|
||||
rows.push({ commodity: p.commodity, price: p.price, unit: p.unit, timestamp: p.timestamp, source: p.source });
|
||||
}
|
||||
log(` FRED coal: ${fredCoal.data.length} records`);
|
||||
} else {
|
||||
log(` ERROR FRED coal: ${fredCoal.error}`);
|
||||
}
|
||||
|
||||
if (rows.length === 0) {
|
||||
log(' No commodity data fetched');
|
||||
return;
|
||||
}
|
||||
|
||||
// Deduplicate: for same commodity + timestamp, prefer EIA over FRED
|
||||
const deduped = new Map<string, CommodityRow>();
|
||||
for (const row of rows) {
|
||||
const key = `${row.commodity}:${row.timestamp.getTime()}`;
|
||||
const existing = deduped.get(key);
|
||||
if (!existing || (existing.source === 'FRED' && row.source === 'EIA')) {
|
||||
deduped.set(key, row);
|
||||
}
|
||||
}
|
||||
const uniqueRows = [...deduped.values()];
|
||||
|
||||
// Upsert into database
|
||||
const timestamps = uniqueRows.map(r => r.timestamp);
|
||||
const commodities = [...new Set(uniqueRows.map(r => r.commodity))];
|
||||
|
||||
const existing = await prisma.commodityPrice.findMany({
|
||||
where: { commodity: { in: commodities }, timestamp: { in: timestamps } },
|
||||
select: { id: true, commodity: true, timestamp: true },
|
||||
});
|
||||
const existingKeys = new Map(existing.map(e => [`${e.commodity}:${e.timestamp.getTime()}`, e.id]));
|
||||
|
||||
const toCreate: Array<{ commodity: string; price: number; unit: string; timestamp: Date; source: string }> = [];
|
||||
const toUpdate: Array<{ id: string; price: number; unit: string; source: string }> = [];
|
||||
|
||||
for (const row of uniqueRows) {
|
||||
const key = `${row.commodity}:${row.timestamp.getTime()}`;
|
||||
const existingId = existingKeys.get(key);
|
||||
if (existingId) {
|
||||
toUpdate.push({ id: existingId, price: row.price, unit: row.unit, source: row.source });
|
||||
} else {
|
||||
toCreate.push({
|
||||
commodity: row.commodity,
|
||||
price: row.price,
|
||||
unit: row.unit,
|
||||
timestamp: row.timestamp,
|
||||
source: row.source,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (toCreate.length > 0) {
|
||||
const result = await prisma.commodityPrice.createMany({ data: toCreate });
|
||||
log(` Commodities: ${result.count} records inserted`);
|
||||
}
|
||||
|
||||
if (toUpdate.length > 0) {
|
||||
const chunkSize = 100;
|
||||
for (let i = 0; i < toUpdate.length; i += chunkSize) {
|
||||
const chunk = toUpdate.slice(i, i + chunkSize);
|
||||
await prisma.$transaction(
|
||||
chunk.map(u =>
|
||||
prisma.commodityPrice.update({
|
||||
where: { id: u.id },
|
||||
data: { price: u.price, unit: u.unit, source: u.source },
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
log(` Commodities: ${toUpdate.length} records updated`);
|
||||
}
|
||||
|
||||
if (toCreate.length === 0 && toUpdate.length === 0) {
|
||||
log(' Commodities: no changes needed');
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function main(): Promise<void> {
|
||||
log('Starting historical backfill (6 months)...');
|
||||
log(`Date range: ${sixMonthsAgoIso()} to ${todayIso()}`);
|
||||
log('');
|
||||
|
||||
await backfillElectricity();
|
||||
log('');
|
||||
|
||||
await backfillGeneration();
|
||||
log('');
|
||||
|
||||
await backfillCommodities();
|
||||
log('');
|
||||
|
||||
log('Backfill complete.');
|
||||
}
|
||||
|
||||
main()
|
||||
.catch(err => {
|
||||
console.error('Backfill failed:', err);
|
||||
process.exit(1);
|
||||
})
|
||||
.finally(() => {
|
||||
void prisma.$disconnect();
|
||||
});
|
||||
74
src/actions/datacenters.ts
Normal file
74
src/actions/datacenters.ts
Normal file
@ -0,0 +1,74 @@
|
||||
'use server';
|
||||
|
||||
import { findDatacentersInRegion, findNearbyDatacenters } from '@/generated/prisma/sql.js';
|
||||
import { prisma } from '@/lib/db.js';
|
||||
import { serialize } from '@/lib/superjson.js';
|
||||
|
||||
interface ActionSuccess<T> {
|
||||
ok: true;
|
||||
data: ReturnType<typeof serialize<T>>;
|
||||
}
|
||||
|
||||
interface ActionError {
|
||||
ok: false;
|
||||
error: string;
|
||||
}
|
||||
|
||||
type ActionResult<T> = ActionSuccess<T> | ActionError;
|
||||
|
||||
export async function fetchDatacenters(): Promise<
|
||||
ActionResult<
|
||||
Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
operator: string;
|
||||
capacityMw: number;
|
||||
status: string;
|
||||
yearOpened: number;
|
||||
regionId: string;
|
||||
createdAt: Date;
|
||||
}>
|
||||
>
|
||||
> {
|
||||
try {
|
||||
const rows = await prisma.datacenter.findMany({
|
||||
orderBy: { capacityMw: 'desc' },
|
||||
});
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch datacenters: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchDatacentersInRegion(
|
||||
regionCode: string,
|
||||
): Promise<ActionResult<findDatacentersInRegion.Result[]>> {
|
||||
try {
|
||||
const rows = await prisma.$queryRawTyped(findDatacentersInRegion(regionCode));
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch datacenters in region ${regionCode}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchNearbyDatacenters(
|
||||
lat: number,
|
||||
lng: number,
|
||||
radiusKm: number,
|
||||
): Promise<ActionResult<findNearbyDatacenters.Result[]>> {
|
||||
try {
|
||||
const rows = await prisma.$queryRawTyped(findNearbyDatacenters(lat, lng, radiusKm));
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch nearby datacenters: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
63
src/actions/demand.ts
Normal file
63
src/actions/demand.ts
Normal file
@ -0,0 +1,63 @@
|
||||
'use server';
|
||||
|
||||
import { getDemandByRegion } from '@/generated/prisma/sql.js';
|
||||
import { prisma } from '@/lib/db.js';
|
||||
import { serialize } from '@/lib/superjson.js';
|
||||
|
||||
type TimeRange = '24h' | '7d' | '30d' | '90d' | '1y';
|
||||
|
||||
function timeRangeToStartDate(range: TimeRange): Date {
|
||||
const now = new Date();
|
||||
const ms: Record<TimeRange, number> = {
|
||||
'24h': 24 * 60 * 60 * 1000,
|
||||
'7d': 7 * 24 * 60 * 60 * 1000,
|
||||
'30d': 30 * 24 * 60 * 60 * 1000,
|
||||
'90d': 90 * 24 * 60 * 60 * 1000,
|
||||
'1y': 365 * 24 * 60 * 60 * 1000,
|
||||
};
|
||||
return new Date(now.getTime() - ms[range]);
|
||||
}
|
||||
|
||||
interface ActionSuccess<T> {
|
||||
ok: true;
|
||||
data: ReturnType<typeof serialize<T>>;
|
||||
}
|
||||
|
||||
interface ActionError {
|
||||
ok: false;
|
||||
error: string;
|
||||
}
|
||||
|
||||
type ActionResult<T> = ActionSuccess<T> | ActionError;
|
||||
|
||||
export async function fetchDemandByRegion(
|
||||
regionCode: string,
|
||||
timeRange: TimeRange = '30d',
|
||||
): Promise<ActionResult<getDemandByRegion.Result[]>> {
|
||||
try {
|
||||
const startDate = timeRangeToStartDate(timeRange);
|
||||
const endDate = new Date();
|
||||
const rows = await prisma.$queryRawTyped(getDemandByRegion(startDate, endDate));
|
||||
const filtered = regionCode === 'ALL' ? rows : rows.filter(r => r.region_code === regionCode);
|
||||
return { ok: true, data: serialize(filtered) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch demand data: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchRegionDemandSummary(): Promise<ActionResult<getDemandByRegion.Result[]>> {
|
||||
try {
|
||||
const startDate = new Date(Date.now() - 7 * 24 * 60 * 60 * 1000);
|
||||
const endDate = new Date();
|
||||
const rows = await prisma.$queryRawTyped(getDemandByRegion(startDate, endDate));
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch region demand summary: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
48
src/actions/generation.ts
Normal file
48
src/actions/generation.ts
Normal file
@ -0,0 +1,48 @@
|
||||
'use server';
|
||||
|
||||
import { getGenerationMix } from '@/generated/prisma/sql.js';
|
||||
import { prisma } from '@/lib/db.js';
|
||||
import { serialize } from '@/lib/superjson.js';
|
||||
|
||||
type TimeRange = '24h' | '7d' | '30d' | '90d' | '1y';
|
||||
|
||||
function timeRangeToStartDate(range: TimeRange): Date {
|
||||
const now = new Date();
|
||||
const ms: Record<TimeRange, number> = {
|
||||
'24h': 24 * 60 * 60 * 1000,
|
||||
'7d': 7 * 24 * 60 * 60 * 1000,
|
||||
'30d': 30 * 24 * 60 * 60 * 1000,
|
||||
'90d': 90 * 24 * 60 * 60 * 1000,
|
||||
'1y': 365 * 24 * 60 * 60 * 1000,
|
||||
};
|
||||
return new Date(now.getTime() - ms[range]);
|
||||
}
|
||||
|
||||
interface ActionSuccess<T> {
|
||||
ok: true;
|
||||
data: ReturnType<typeof serialize<T>>;
|
||||
}
|
||||
|
||||
interface ActionError {
|
||||
ok: false;
|
||||
error: string;
|
||||
}
|
||||
|
||||
type ActionResult<T> = ActionSuccess<T> | ActionError;
|
||||
|
||||
export async function fetchGenerationMix(
|
||||
regionCode: string,
|
||||
timeRange: TimeRange = '30d',
|
||||
): Promise<ActionResult<getGenerationMix.Result[]>> {
|
||||
try {
|
||||
const startDate = timeRangeToStartDate(timeRange);
|
||||
const endDate = new Date();
|
||||
const rows = await prisma.$queryRawTyped(getGenerationMix(regionCode, startDate, endDate));
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch generation mix: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
97
src/actions/prices.ts
Normal file
97
src/actions/prices.ts
Normal file
@ -0,0 +1,97 @@
|
||||
'use server';
|
||||
|
||||
import { getLatestPrices, getPriceTrends, getRegionPriceHeatmap } from '@/generated/prisma/sql.js';
|
||||
import { prisma } from '@/lib/db.js';
|
||||
import { serialize } from '@/lib/superjson.js';
|
||||
|
||||
type TimeRange = '24h' | '7d' | '30d' | '90d' | '1y';
|
||||
|
||||
function timeRangeToStartDate(range: TimeRange): Date {
|
||||
const now = new Date();
|
||||
const ms: Record<TimeRange, number> = {
|
||||
'24h': 24 * 60 * 60 * 1000,
|
||||
'7d': 7 * 24 * 60 * 60 * 1000,
|
||||
'30d': 30 * 24 * 60 * 60 * 1000,
|
||||
'90d': 90 * 24 * 60 * 60 * 1000,
|
||||
'1y': 365 * 24 * 60 * 60 * 1000,
|
||||
};
|
||||
return new Date(now.getTime() - ms[range]);
|
||||
}
|
||||
|
||||
interface ActionSuccess<T> {
|
||||
ok: true;
|
||||
data: ReturnType<typeof serialize<T>>;
|
||||
}
|
||||
|
||||
interface ActionError {
|
||||
ok: false;
|
||||
error: string;
|
||||
}
|
||||
|
||||
type ActionResult<T> = ActionSuccess<T> | ActionError;
|
||||
|
||||
export async function fetchLatestPrices(): Promise<ActionResult<getLatestPrices.Result[]>> {
|
||||
try {
|
||||
const rows = await prisma.$queryRawTyped(getLatestPrices());
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch latest prices: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchPriceTrends(
|
||||
regionCode: string,
|
||||
timeRange: TimeRange = '30d',
|
||||
): Promise<ActionResult<getPriceTrends.Result[]>> {
|
||||
try {
|
||||
const startDate = timeRangeToStartDate(timeRange);
|
||||
const endDate = new Date();
|
||||
const rows = await prisma.$queryRawTyped(getPriceTrends(regionCode, startDate, endDate));
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch price trends: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchPriceHeatmapData(): Promise<ActionResult<getRegionPriceHeatmap.Result[]>> {
|
||||
try {
|
||||
const rows = await prisma.$queryRawTyped(getRegionPriceHeatmap());
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch price heatmap data: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchLatestCommodityPrices(): Promise<
|
||||
ActionResult<
|
||||
Array<{
|
||||
commodity: string;
|
||||
price: number;
|
||||
unit: string;
|
||||
timestamp: Date;
|
||||
source: string;
|
||||
}>
|
||||
>
|
||||
> {
|
||||
try {
|
||||
const rows = await prisma.commodityPrice.findMany({
|
||||
orderBy: { timestamp: 'desc' },
|
||||
distinct: ['commodity'],
|
||||
});
|
||||
return { ok: true, data: serialize(rows) };
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `Failed to fetch commodity prices: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
198
src/app/api/ingest/commodities/route.ts
Normal file
198
src/app/api/ingest/commodities/route.ts
Normal file
@ -0,0 +1,198 @@
|
||||
import { NextResponse, type NextRequest } from 'next/server.js';
|
||||
|
||||
import * as eia from '@/lib/api/eia.js';
|
||||
import * as fred from '@/lib/api/fred.js';
|
||||
import { prisma } from '@/lib/db.js';
|
||||
|
||||
interface IngestionStats {
|
||||
inserted: number;
|
||||
updated: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
interface CommodityRow {
|
||||
commodity: string;
|
||||
price: number;
|
||||
unit: string;
|
||||
timestamp: Date;
|
||||
source: string;
|
||||
}
|
||||
|
||||
async function fetchAllCommodities(start?: string, end?: string): Promise<{ rows: CommodityRow[]; errors: number }> {
|
||||
const rows: CommodityRow[] = [];
|
||||
let errors = 0;
|
||||
|
||||
// EIA: Natural Gas
|
||||
try {
|
||||
const gasData = await eia.getNaturalGasPrice({ start, end });
|
||||
for (const p of gasData) {
|
||||
if (p.price === null) continue;
|
||||
rows.push({
|
||||
commodity: p.commodity,
|
||||
price: p.price,
|
||||
unit: p.unit,
|
||||
timestamp: p.timestamp,
|
||||
source: p.source,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch EIA natural gas:', err);
|
||||
errors++;
|
||||
}
|
||||
|
||||
// EIA: WTI Crude
|
||||
try {
|
||||
const oilData = await eia.getWTICrudePrice({ start, end });
|
||||
for (const p of oilData) {
|
||||
if (p.price === null) continue;
|
||||
rows.push({
|
||||
commodity: p.commodity,
|
||||
price: p.price,
|
||||
unit: p.unit,
|
||||
timestamp: p.timestamp,
|
||||
source: p.source,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch EIA WTI crude:', err);
|
||||
errors++;
|
||||
}
|
||||
|
||||
// FRED: Natural Gas (DHHNGSP)
|
||||
const fredGas = await fred.getNaturalGasPrice(start, end);
|
||||
if (fredGas.ok) {
|
||||
for (const p of fredGas.data) {
|
||||
rows.push({
|
||||
commodity: p.commodity,
|
||||
price: p.price,
|
||||
unit: p.unit,
|
||||
timestamp: p.timestamp,
|
||||
source: p.source,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.error('Failed to fetch FRED natural gas:', fredGas.error);
|
||||
errors++;
|
||||
}
|
||||
|
||||
// FRED: WTI Crude (DCOILWTICO)
|
||||
const fredOil = await fred.getWTICrudePrice(start, end);
|
||||
if (fredOil.ok) {
|
||||
for (const p of fredOil.data) {
|
||||
rows.push({
|
||||
commodity: p.commodity,
|
||||
price: p.price,
|
||||
unit: p.unit,
|
||||
timestamp: p.timestamp,
|
||||
source: p.source,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.error('Failed to fetch FRED WTI crude:', fredOil.error);
|
||||
errors++;
|
||||
}
|
||||
|
||||
// FRED: Coal (PCOALAUUSDM — monthly only)
|
||||
const fredCoal = await fred.getCoalPrice(start, end);
|
||||
if (fredCoal.ok) {
|
||||
for (const p of fredCoal.data) {
|
||||
rows.push({
|
||||
commodity: p.commodity,
|
||||
price: p.price,
|
||||
unit: p.unit,
|
||||
timestamp: p.timestamp,
|
||||
source: p.source,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.error('Failed to fetch FRED coal:', fredCoal.error);
|
||||
errors++;
|
||||
}
|
||||
|
||||
return { rows, errors };
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest): Promise<NextResponse<IngestionStats>> {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const start = searchParams.get('start') ?? undefined;
|
||||
const end = searchParams.get('end') ?? undefined;
|
||||
|
||||
const stats: IngestionStats = { inserted: 0, updated: 0, errors: 0 };
|
||||
|
||||
const { rows, errors: fetchErrors } = await fetchAllCommodities(start, end);
|
||||
stats.errors += fetchErrors;
|
||||
|
||||
if (rows.length === 0) {
|
||||
return NextResponse.json(stats);
|
||||
}
|
||||
|
||||
// Deduplicate: for same commodity + timestamp, prefer EIA over FRED
|
||||
const deduped = new Map<string, CommodityRow>();
|
||||
for (const row of rows) {
|
||||
const key = `${row.commodity}:${row.timestamp.getTime()}`;
|
||||
const existing = deduped.get(key);
|
||||
if (!existing || (existing.source === 'FRED' && row.source === 'EIA')) {
|
||||
deduped.set(key, row);
|
||||
}
|
||||
}
|
||||
|
||||
const uniqueRows = [...deduped.values()];
|
||||
|
||||
// Fetch existing records to determine inserts vs updates
|
||||
const timestamps = uniqueRows.map(r => r.timestamp);
|
||||
const commodities = [...new Set(uniqueRows.map(r => r.commodity))];
|
||||
|
||||
const existing = await prisma.commodityPrice.findMany({
|
||||
where: {
|
||||
commodity: { in: commodities },
|
||||
timestamp: { in: timestamps },
|
||||
},
|
||||
select: { id: true, commodity: true, timestamp: true },
|
||||
});
|
||||
|
||||
const existingKeys = new Map(existing.map(e => [`${e.commodity}:${e.timestamp.getTime()}`, e.id]));
|
||||
|
||||
const toCreate: Array<{
|
||||
commodity: string;
|
||||
price: number;
|
||||
unit: string;
|
||||
timestamp: Date;
|
||||
source: string;
|
||||
}> = [];
|
||||
const toUpdate: Array<{ id: string; price: number; unit: string; source: string }> = [];
|
||||
|
||||
for (const row of uniqueRows) {
|
||||
const key = `${row.commodity}:${row.timestamp.getTime()}`;
|
||||
const existingId = existingKeys.get(key);
|
||||
if (existingId) {
|
||||
toUpdate.push({ id: existingId, price: row.price, unit: row.unit, source: row.source });
|
||||
} else {
|
||||
toCreate.push({
|
||||
commodity: row.commodity,
|
||||
price: row.price,
|
||||
unit: row.unit,
|
||||
timestamp: row.timestamp,
|
||||
source: row.source,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (toCreate.length > 0) {
|
||||
const result = await prisma.commodityPrice.createMany({ data: toCreate });
|
||||
stats.inserted += result.count;
|
||||
}
|
||||
|
||||
if (toUpdate.length > 0) {
|
||||
await prisma.$transaction(
|
||||
toUpdate.map(u =>
|
||||
prisma.commodityPrice.update({
|
||||
where: { id: u.id },
|
||||
data: { price: u.price, unit: u.unit, source: u.source },
|
||||
}),
|
||||
),
|
||||
);
|
||||
stats.updated += toUpdate.length;
|
||||
}
|
||||
|
||||
return NextResponse.json(stats);
|
||||
}
|
||||
116
src/app/api/ingest/electricity/route.ts
Normal file
116
src/app/api/ingest/electricity/route.ts
Normal file
@ -0,0 +1,116 @@
|
||||
import { NextResponse, type NextRequest } from 'next/server.js';
|
||||
|
||||
import { getRegionData } from '@/lib/api/eia.js';
|
||||
import { prisma } from '@/lib/db.js';
|
||||
import { EIA_RESPONDENT_CODES, type RegionCode } from '@/lib/schemas/electricity.js';
|
||||
|
||||
const ALL_REGIONS: RegionCode[] = ['PJM', 'ERCOT', 'CAISO', 'NYISO', 'ISONE', 'MISO', 'SPP'];
|
||||
|
||||
function isRegionCode(value: string): value is RegionCode {
|
||||
return value in EIA_RESPONDENT_CODES;
|
||||
}
|
||||
|
||||
interface IngestionStats {
|
||||
inserted: number;
|
||||
updated: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest): Promise<NextResponse<IngestionStats>> {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const regionParam = searchParams.get('region');
|
||||
const start = searchParams.get('start') ?? undefined;
|
||||
const end = searchParams.get('end') ?? undefined;
|
||||
|
||||
let regions: RegionCode[];
|
||||
if (regionParam) {
|
||||
if (!isRegionCode(regionParam)) {
|
||||
return NextResponse.json({ inserted: 0, updated: 0, errors: 1 }, { status: 400 });
|
||||
}
|
||||
regions = [regionParam];
|
||||
} else {
|
||||
regions = ALL_REGIONS;
|
||||
}
|
||||
|
||||
const stats: IngestionStats = { inserted: 0, updated: 0, errors: 0 };
|
||||
|
||||
// Pre-fetch all grid regions in one query
|
||||
const gridRegions = await prisma.gridRegion.findMany({
|
||||
where: { code: { in: regions } },
|
||||
select: { id: true, code: true },
|
||||
});
|
||||
const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id]));
|
||||
|
||||
for (const regionCode of regions) {
|
||||
const regionId = regionIdByCode.get(regionCode);
|
||||
if (!regionId) {
|
||||
stats.errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const demandData = await getRegionData(regionCode, 'D', { start, end });
|
||||
const validPoints = demandData.filter((p): p is typeof p & { valueMw: number } => p.valueMw !== null);
|
||||
|
||||
if (validPoints.length === 0) continue;
|
||||
|
||||
// Fetch existing records for this region in the time range to avoid N+1
|
||||
const timestamps = validPoints.map(p => p.timestamp);
|
||||
const existing = await prisma.electricityPrice.findMany({
|
||||
where: {
|
||||
regionId,
|
||||
timestamp: { in: timestamps },
|
||||
},
|
||||
select: { id: true, timestamp: true },
|
||||
});
|
||||
const existingByTime = new Map(existing.map(e => [e.timestamp.getTime(), e.id]));
|
||||
|
||||
const toCreate: Array<{
|
||||
regionId: string;
|
||||
priceMwh: number;
|
||||
demandMw: number;
|
||||
timestamp: Date;
|
||||
source: string;
|
||||
}> = [];
|
||||
const toUpdate: Array<{ id: string; demandMw: number }> = [];
|
||||
|
||||
for (const point of validPoints) {
|
||||
const existingId = existingByTime.get(point.timestamp.getTime());
|
||||
if (existingId) {
|
||||
toUpdate.push({ id: existingId, demandMw: point.valueMw });
|
||||
} else {
|
||||
toCreate.push({
|
||||
regionId,
|
||||
priceMwh: 0,
|
||||
demandMw: point.valueMw,
|
||||
timestamp: point.timestamp,
|
||||
source: 'EIA',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (toCreate.length > 0) {
|
||||
const result = await prisma.electricityPrice.createMany({ data: toCreate });
|
||||
stats.inserted += result.count;
|
||||
}
|
||||
|
||||
// Batch updates via transaction
|
||||
if (toUpdate.length > 0) {
|
||||
await prisma.$transaction(
|
||||
toUpdate.map(u =>
|
||||
prisma.electricityPrice.update({
|
||||
where: { id: u.id },
|
||||
data: { demandMw: u.demandMw, source: 'EIA' },
|
||||
}),
|
||||
),
|
||||
);
|
||||
stats.updated += toUpdate.length;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Failed to ingest electricity data for ${regionCode}:`, err);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json(stats);
|
||||
}
|
||||
114
src/app/api/ingest/generation/route.ts
Normal file
114
src/app/api/ingest/generation/route.ts
Normal file
@ -0,0 +1,114 @@
|
||||
import { NextResponse, type NextRequest } from 'next/server.js';
|
||||
|
||||
import { getFuelTypeData } from '@/lib/api/eia.js';
|
||||
import { prisma } from '@/lib/db.js';
|
||||
import { EIA_RESPONDENT_CODES, type RegionCode } from '@/lib/schemas/electricity.js';
|
||||
|
||||
const ALL_REGIONS: RegionCode[] = ['PJM', 'ERCOT', 'CAISO', 'NYISO', 'ISONE', 'MISO', 'SPP'];
|
||||
|
||||
function isRegionCode(value: string): value is RegionCode {
|
||||
return value in EIA_RESPONDENT_CODES;
|
||||
}
|
||||
|
||||
interface IngestionStats {
|
||||
inserted: number;
|
||||
updated: number;
|
||||
errors: number;
|
||||
}
|
||||
|
||||
export async function GET(request: NextRequest): Promise<NextResponse<IngestionStats>> {
|
||||
const searchParams = request.nextUrl.searchParams;
|
||||
const regionParam = searchParams.get('region');
|
||||
const start = searchParams.get('start') ?? undefined;
|
||||
const end = searchParams.get('end') ?? undefined;
|
||||
|
||||
let regions: RegionCode[];
|
||||
if (regionParam) {
|
||||
if (!isRegionCode(regionParam)) {
|
||||
return NextResponse.json({ inserted: 0, updated: 0, errors: 1 }, { status: 400 });
|
||||
}
|
||||
regions = [regionParam];
|
||||
} else {
|
||||
regions = ALL_REGIONS;
|
||||
}
|
||||
|
||||
const stats: IngestionStats = { inserted: 0, updated: 0, errors: 0 };
|
||||
|
||||
// Pre-fetch all grid regions in one query
|
||||
const gridRegions = await prisma.gridRegion.findMany({
|
||||
where: { code: { in: regions } },
|
||||
select: { id: true, code: true },
|
||||
});
|
||||
const regionIdByCode = new Map(gridRegions.map(r => [r.code, r.id]));
|
||||
|
||||
for (const regionCode of regions) {
|
||||
const regionId = regionIdByCode.get(regionCode);
|
||||
if (!regionId) {
|
||||
stats.errors++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const fuelData = await getFuelTypeData(regionCode, { start, end });
|
||||
const validPoints = fuelData.filter((p): p is typeof p & { generationMw: number } => p.generationMw !== null);
|
||||
|
||||
if (validPoints.length === 0) continue;
|
||||
|
||||
// Fetch existing records for this region to determine inserts vs updates
|
||||
const timestamps = validPoints.map(p => p.timestamp);
|
||||
const existing = await prisma.generationMix.findMany({
|
||||
where: {
|
||||
regionId,
|
||||
timestamp: { in: timestamps },
|
||||
},
|
||||
select: { id: true, timestamp: true, fuelType: true },
|
||||
});
|
||||
const existingKeys = new Map(existing.map(e => [`${e.fuelType}:${e.timestamp.getTime()}`, e.id]));
|
||||
|
||||
const toCreate: Array<{
|
||||
regionId: string;
|
||||
fuelType: string;
|
||||
generationMw: number;
|
||||
timestamp: Date;
|
||||
}> = [];
|
||||
const toUpdate: Array<{ id: string; generationMw: number }> = [];
|
||||
|
||||
for (const point of validPoints) {
|
||||
const key = `${point.fuelType}:${point.timestamp.getTime()}`;
|
||||
const existingId = existingKeys.get(key);
|
||||
if (existingId) {
|
||||
toUpdate.push({ id: existingId, generationMw: point.generationMw });
|
||||
} else {
|
||||
toCreate.push({
|
||||
regionId,
|
||||
fuelType: point.fuelType,
|
||||
generationMw: point.generationMw,
|
||||
timestamp: point.timestamp,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (toCreate.length > 0) {
|
||||
const result = await prisma.generationMix.createMany({ data: toCreate });
|
||||
stats.inserted += result.count;
|
||||
}
|
||||
|
||||
if (toUpdate.length > 0) {
|
||||
await prisma.$transaction(
|
||||
toUpdate.map(u =>
|
||||
prisma.generationMix.update({
|
||||
where: { id: u.id },
|
||||
data: { generationMw: u.generationMw },
|
||||
}),
|
||||
),
|
||||
);
|
||||
stats.updated += toUpdate.length;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`Failed to ingest generation data for ${regionCode}:`, err);
|
||||
stats.errors++;
|
||||
}
|
||||
}
|
||||
|
||||
return NextResponse.json(stats);
|
||||
}
|
||||
339
src/lib/api/eia.ts
Normal file
339
src/lib/api/eia.ts
Normal file
@ -0,0 +1,339 @@
|
||||
import {
|
||||
type CommodityPricePoint,
|
||||
eiaNaturalGasResponseSchema,
|
||||
eiaWtiCrudeResponseSchema,
|
||||
parseEiaCommodityPeriod,
|
||||
} from '@/lib/schemas/commodities.js';
|
||||
import {
|
||||
EIA_RESPONDENT_CODES,
|
||||
type EiaFuelTypeDataRow,
|
||||
type EiaRegionDataRow,
|
||||
type FuelTypeDataPoint,
|
||||
type RegionCode,
|
||||
type RegionDataPoint,
|
||||
eiaFuelTypeDataResponseSchema,
|
||||
eiaRegionDataResponseSchema,
|
||||
parseEiaPeriod,
|
||||
resolveRegionCode,
|
||||
} from '@/lib/schemas/electricity.js';
|
||||
|
||||
const EIA_BASE_URL = 'https://api.eia.gov/v2';
|
||||
const MAX_ROWS_PER_REQUEST = 5000;
|
||||
|
||||
function getApiKey(): string {
|
||||
const key = process.env.EIA_API_KEY;
|
||||
if (!key) {
|
||||
throw new Error('EIA_API_KEY environment variable is not set');
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
interface EiaQueryParams {
|
||||
frequency?: 'hourly' | 'daily' | 'weekly' | 'monthly' | 'annual';
|
||||
start?: string;
|
||||
end?: string;
|
||||
facets?: Record<string, string[]>;
|
||||
sort?: Array<{ column: string; direction: 'asc' | 'desc' }>;
|
||||
offset?: number;
|
||||
length?: number;
|
||||
}
|
||||
|
||||
function buildUrl(endpoint: string, params: EiaQueryParams): string {
|
||||
const url = new URL(`${EIA_BASE_URL}${endpoint}`);
|
||||
url.searchParams.set('api_key', getApiKey());
|
||||
url.searchParams.set('data[0]', 'value');
|
||||
|
||||
if (params.frequency) {
|
||||
url.searchParams.set('frequency', params.frequency);
|
||||
}
|
||||
if (params.start) {
|
||||
url.searchParams.set('start', params.start);
|
||||
}
|
||||
if (params.end) {
|
||||
url.searchParams.set('end', params.end);
|
||||
}
|
||||
if (params.facets) {
|
||||
for (const [key, values] of Object.entries(params.facets)) {
|
||||
for (const value of values) {
|
||||
url.searchParams.append(`facets[${key}][]`, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (params.sort) {
|
||||
for (let i = 0; i < params.sort.length; i++) {
|
||||
const s = params.sort[i];
|
||||
if (s) {
|
||||
url.searchParams.set(`sort[${i}][column]`, s.column);
|
||||
url.searchParams.set(`sort[${i}][direction]`, s.direction);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
url.searchParams.set('offset', String(params.offset ?? 0));
|
||||
url.searchParams.set('length', String(params.length ?? MAX_ROWS_PER_REQUEST));
|
||||
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
async function fetchEia(endpoint: string, params: EiaQueryParams): Promise<unknown> {
|
||||
const url = buildUrl(endpoint, params);
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
const text = await response.text().catch(() => 'unknown error');
|
||||
throw new Error(`EIA API error ${response.status}: ${text}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-paginate through an EIA endpoint, collecting all rows.
|
||||
* EIA limits responses to 5000 rows — this fetches all pages sequentially.
|
||||
*/
|
||||
async function fetchAllPages<T>(
|
||||
endpoint: string,
|
||||
params: EiaQueryParams,
|
||||
parseResponse: (json: unknown) => { total: number; data: T[] },
|
||||
): Promise<T[]> {
|
||||
const allData: T[] = [];
|
||||
let offset = params.offset ?? 0;
|
||||
const length = params.length ?? MAX_ROWS_PER_REQUEST;
|
||||
|
||||
for (;;) {
|
||||
const json = await fetchEia(endpoint, { ...params, offset, length });
|
||||
const result = parseResponse(json);
|
||||
|
||||
allData.push(...result.data);
|
||||
|
||||
if (allData.length >= result.total || result.data.length < length) {
|
||||
break;
|
||||
}
|
||||
|
||||
offset += length;
|
||||
}
|
||||
|
||||
return allData;
|
||||
}
|
||||
|
||||
/** Convert a raw EIA region-data row into a typed RegionDataPoint */
|
||||
function transformRegionDataRow(row: EiaRegionDataRow): RegionDataPoint {
|
||||
const regionCode = resolveRegionCode(row.respondent);
|
||||
|
||||
return {
|
||||
timestamp: parseEiaPeriod(row.period),
|
||||
regionCode: regionCode ?? 'PJM',
|
||||
respondent: row.respondent,
|
||||
type: row.type,
|
||||
typeName: row['type-name'],
|
||||
valueMw: row.value,
|
||||
valueUnits: row['value-units'],
|
||||
};
|
||||
}
|
||||
|
||||
/** Convert a raw EIA fuel-type-data row into a typed FuelTypeDataPoint */
|
||||
function transformFuelTypeRow(row: EiaFuelTypeDataRow): FuelTypeDataPoint {
|
||||
const regionCode = resolveRegionCode(row.respondent);
|
||||
|
||||
return {
|
||||
timestamp: parseEiaPeriod(row.period),
|
||||
regionCode: regionCode ?? 'PJM',
|
||||
respondent: row.respondent,
|
||||
fuelType: row.fueltype,
|
||||
typeName: row['type-name'],
|
||||
generationMw: row.value,
|
||||
valueUnits: row['value-units'],
|
||||
};
|
||||
}
|
||||
|
||||
export interface GetRegionDataOptions {
|
||||
start?: string;
|
||||
end?: string;
|
||||
/** Maximum number of rows to fetch. Omit for all available data. */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch hourly demand or net generation data for a region.
|
||||
*
|
||||
* @param regionCode - One of PJM, ERCOT, CAISO, NYISO, ISONE, MISO, SPP
|
||||
* @param type - "D" for demand, "NG" for net generation
|
||||
*
|
||||
* NOTE: EIA does not provide real-time wholesale electricity prices.
|
||||
* The /v2/electricity/rto/region-data/data/ endpoint provides demand (MW) and
|
||||
* net generation (MW), NOT price ($/MWh). For the electricity_prices table,
|
||||
* we store demand_mw from this data; price_mwh will be null.
|
||||
* TODO: Investigate scraping ISO-specific price feeds (PJM LMP, ERCOT SPP, CAISO LMP)
|
||||
* for real wholesale price data if needed in the future.
|
||||
*/
|
||||
export async function getRegionData(
|
||||
regionCode: RegionCode,
|
||||
type: 'D' | 'NG',
|
||||
options: GetRegionDataOptions = {},
|
||||
): Promise<RegionDataPoint[]> {
|
||||
const respondentCode = EIA_RESPONDENT_CODES[regionCode];
|
||||
|
||||
const params: EiaQueryParams = {
|
||||
frequency: 'hourly',
|
||||
start: options.start,
|
||||
end: options.end,
|
||||
facets: {
|
||||
respondent: [respondentCode],
|
||||
type: [type],
|
||||
},
|
||||
sort: [{ column: 'period', direction: 'desc' }],
|
||||
length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST,
|
||||
};
|
||||
|
||||
if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) {
|
||||
const json = await fetchEia('/electricity/rto/region-data/data/', params);
|
||||
const parsed = eiaRegionDataResponseSchema.parse(json);
|
||||
return parsed.response.data.map(transformRegionDataRow);
|
||||
}
|
||||
|
||||
const rows = await fetchAllPages('/electricity/rto/region-data/data/', params, json => {
|
||||
const parsed = eiaRegionDataResponseSchema.parse(json);
|
||||
return { total: parsed.response.total, data: parsed.response.data };
|
||||
});
|
||||
|
||||
return rows.map(transformRegionDataRow);
|
||||
}
|
||||
|
||||
export interface GetFuelTypeDataOptions {
|
||||
start?: string;
|
||||
end?: string;
|
||||
fuelType?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch hourly generation by fuel type for a region.
|
||||
*
|
||||
* @param regionCode - One of PJM, ERCOT, CAISO, NYISO, ISONE, MISO, SPP
|
||||
* @param options - Optional filters for time range and fuel type
|
||||
*/
|
||||
export async function getFuelTypeData(
|
||||
regionCode: RegionCode,
|
||||
options: GetFuelTypeDataOptions = {},
|
||||
): Promise<FuelTypeDataPoint[]> {
|
||||
const respondentCode = EIA_RESPONDENT_CODES[regionCode];
|
||||
|
||||
const facets: Record<string, string[]> = {
|
||||
respondent: [respondentCode],
|
||||
};
|
||||
if (options.fuelType) {
|
||||
facets['fueltype'] = [options.fuelType];
|
||||
}
|
||||
|
||||
const params: EiaQueryParams = {
|
||||
frequency: 'hourly',
|
||||
start: options.start,
|
||||
end: options.end,
|
||||
facets,
|
||||
sort: [{ column: 'period', direction: 'desc' }],
|
||||
length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST,
|
||||
};
|
||||
|
||||
if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) {
|
||||
const json = await fetchEia('/electricity/rto/fuel-type-data/data/', params);
|
||||
const parsed = eiaFuelTypeDataResponseSchema.parse(json);
|
||||
return parsed.response.data.map(transformFuelTypeRow);
|
||||
}
|
||||
|
||||
const rows = await fetchAllPages('/electricity/rto/fuel-type-data/data/', params, json => {
|
||||
const parsed = eiaFuelTypeDataResponseSchema.parse(json);
|
||||
return { total: parsed.response.total, data: parsed.response.data };
|
||||
});
|
||||
|
||||
return rows.map(transformFuelTypeRow);
|
||||
}
|
||||
|
||||
export interface GetCommodityPriceOptions {
|
||||
start?: string;
|
||||
end?: string;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch Henry Hub natural gas spot prices.
|
||||
* Endpoint: /v2/natural-gas/pri/fut/data/ with facets[series][]=RNGWHHD
|
||||
*/
|
||||
export async function getNaturalGasPrice(options: GetCommodityPriceOptions = {}): Promise<CommodityPricePoint[]> {
|
||||
const params: EiaQueryParams = {
|
||||
frequency: 'daily',
|
||||
start: options.start,
|
||||
end: options.end,
|
||||
facets: {
|
||||
series: ['RNGWHHD'],
|
||||
},
|
||||
sort: [{ column: 'period', direction: 'desc' }],
|
||||
length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST,
|
||||
};
|
||||
|
||||
if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) {
|
||||
const json = await fetchEia('/natural-gas/pri/fut/data/', params);
|
||||
const parsed = eiaNaturalGasResponseSchema.parse(json);
|
||||
return parsed.response.data.map(row => ({
|
||||
timestamp: parseEiaCommodityPeriod(row.period),
|
||||
commodity: 'natural_gas' as const,
|
||||
price: row.value,
|
||||
unit: row.units ?? '$/Million BTU',
|
||||
source: 'EIA',
|
||||
}));
|
||||
}
|
||||
|
||||
const rows = await fetchAllPages('/natural-gas/pri/fut/data/', params, json => {
|
||||
const parsed = eiaNaturalGasResponseSchema.parse(json);
|
||||
return { total: parsed.response.total, data: parsed.response.data };
|
||||
});
|
||||
|
||||
return rows.map(row => ({
|
||||
timestamp: parseEiaCommodityPeriod(row.period),
|
||||
commodity: 'natural_gas' as const,
|
||||
price: row.value,
|
||||
unit: row.units ?? '$/Million BTU',
|
||||
source: 'EIA',
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch WTI crude oil spot prices.
|
||||
* Endpoint: /v2/petroleum/pri/spt/data/ with facets[series][]=RWTC
|
||||
*/
|
||||
export async function getWTICrudePrice(options: GetCommodityPriceOptions = {}): Promise<CommodityPricePoint[]> {
|
||||
const params: EiaQueryParams = {
|
||||
frequency: 'daily',
|
||||
start: options.start,
|
||||
end: options.end,
|
||||
facets: {
|
||||
series: ['RWTC'],
|
||||
},
|
||||
sort: [{ column: 'period', direction: 'desc' }],
|
||||
length: options.limit ? Math.min(options.limit, MAX_ROWS_PER_REQUEST) : MAX_ROWS_PER_REQUEST,
|
||||
};
|
||||
|
||||
if (options.limit && options.limit <= MAX_ROWS_PER_REQUEST) {
|
||||
const json = await fetchEia('/petroleum/pri/spt/data/', params);
|
||||
const parsed = eiaWtiCrudeResponseSchema.parse(json);
|
||||
return parsed.response.data.map(row => ({
|
||||
timestamp: parseEiaCommodityPeriod(row.period),
|
||||
commodity: 'wti_crude' as const,
|
||||
price: row.value,
|
||||
unit: row.units ?? '$/Barrel',
|
||||
source: 'EIA',
|
||||
}));
|
||||
}
|
||||
|
||||
const rows = await fetchAllPages('/petroleum/pri/spt/data/', params, json => {
|
||||
const parsed = eiaWtiCrudeResponseSchema.parse(json);
|
||||
return { total: parsed.response.total, data: parsed.response.data };
|
||||
});
|
||||
|
||||
return rows.map(row => ({
|
||||
timestamp: parseEiaCommodityPeriod(row.period),
|
||||
commodity: 'wti_crude' as const,
|
||||
price: row.value,
|
||||
unit: row.units ?? '$/Barrel',
|
||||
source: 'EIA',
|
||||
}));
|
||||
}
|
||||
248
src/lib/api/fred.ts
Normal file
248
src/lib/api/fred.ts
Normal file
@ -0,0 +1,248 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
import { COMMODITY_UNITS, type CommodityPrice, type CommodityType } from '@/lib/schemas/commodities.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// FRED Series IDs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FRED_SERIES = {
|
||||
natural_gas: 'DHHNGSP',
|
||||
wti_crude: 'DCOILWTICO',
|
||||
coal: 'PCOALAUUSDM',
|
||||
} as const satisfies Record<CommodityType, string>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Zod Schemas — FRED API Response
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FredObservationSchema = z.object({
|
||||
realtime_start: z.string(),
|
||||
realtime_end: z.string(),
|
||||
date: z.string(), // "YYYY-MM-DD"
|
||||
value: z.string(), // numeric string or "." for missing
|
||||
});
|
||||
|
||||
const FredSeriesResponseSchema = z.object({
|
||||
realtime_start: z.string(),
|
||||
realtime_end: z.string(),
|
||||
observation_start: z.string(),
|
||||
observation_end: z.string(),
|
||||
units: z.string(),
|
||||
output_type: z.number(),
|
||||
file_type: z.string(),
|
||||
order_by: z.string(),
|
||||
sort_order: z.string(),
|
||||
count: z.number(),
|
||||
offset: z.number(),
|
||||
limit: z.number(),
|
||||
observations: z.array(FredObservationSchema),
|
||||
});
|
||||
|
||||
export type FredObservation = z.infer<typeof FredObservationSchema>;
|
||||
export type FredSeriesResponse = z.infer<typeof FredSeriesResponseSchema>;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parsed observation (after filtering missing "." values)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface FredDataPoint {
|
||||
date: Date; // UTC midnight for the observation date
|
||||
value: number;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Error type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export interface FredApiError {
|
||||
ok: false;
|
||||
error: string;
|
||||
status?: number;
|
||||
}
|
||||
|
||||
export interface FredApiSuccess<T> {
|
||||
ok: true;
|
||||
data: T;
|
||||
}
|
||||
|
||||
export type FredApiResult<T> = FredApiSuccess<T> | FredApiError;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Rate limiter — 50ms delay between requests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
let lastRequestTime = 0;
|
||||
|
||||
async function rateLimitDelay(): Promise<void> {
|
||||
const now = Date.now();
|
||||
const elapsed = now - lastRequestTime;
|
||||
if (elapsed < 50) {
|
||||
await new Promise(resolve => setTimeout(resolve, 50 - elapsed));
|
||||
}
|
||||
lastRequestTime = Date.now();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Core fetcher
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const FRED_BASE_URL = 'https://api.stlouisfed.org/fred/series/observations';
|
||||
|
||||
export interface GetSeriesOptions {
|
||||
observationStart?: string; // "YYYY-MM-DD"
|
||||
observationEnd?: string; // "YYYY-MM-DD"
|
||||
limit?: number; // default 100000 (FRED max)
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export async function getSeriesObservations(
|
||||
seriesId: string,
|
||||
options: GetSeriesOptions = {},
|
||||
): Promise<FredApiResult<FredDataPoint[]>> {
|
||||
const apiKey = process.env.FRED_API_KEY;
|
||||
if (!apiKey) {
|
||||
return { ok: false, error: 'FRED_API_KEY is not set' };
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({
|
||||
series_id: seriesId,
|
||||
api_key: apiKey,
|
||||
file_type: 'json',
|
||||
});
|
||||
|
||||
if (options.observationStart) {
|
||||
params.set('observation_start', options.observationStart);
|
||||
}
|
||||
if (options.observationEnd) {
|
||||
params.set('observation_end', options.observationEnd);
|
||||
}
|
||||
if (options.limit !== undefined) {
|
||||
params.set('limit', String(options.limit));
|
||||
}
|
||||
if (options.offset !== undefined) {
|
||||
params.set('offset', String(options.offset));
|
||||
}
|
||||
|
||||
await rateLimitDelay();
|
||||
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(`${FRED_BASE_URL}?${params.toString()}`);
|
||||
} catch (err) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `FRED API network error: ${err instanceof Error ? err.message : String(err)}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text().catch(() => '');
|
||||
return {
|
||||
ok: false,
|
||||
error: `FRED API returned ${response.status}: ${body}`,
|
||||
status: response.status,
|
||||
};
|
||||
}
|
||||
|
||||
let json: unknown;
|
||||
try {
|
||||
json = await response.json();
|
||||
} catch {
|
||||
return { ok: false, error: 'FRED API returned invalid JSON' };
|
||||
}
|
||||
|
||||
const parsed = FredSeriesResponseSchema.safeParse(json);
|
||||
if (!parsed.success) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `FRED response validation failed: ${parsed.error.message}`,
|
||||
};
|
||||
}
|
||||
|
||||
const dataPoints = parseObservations(parsed.data.observations);
|
||||
|
||||
return { ok: true, data: dataPoints };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Parse observations — filter out missing values (".")
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function parseObservations(observations: FredObservation[]): FredDataPoint[] {
|
||||
const points: FredDataPoint[] = [];
|
||||
|
||||
for (const obs of observations) {
|
||||
if (obs.value === '.') continue;
|
||||
|
||||
const numValue = Number(obs.value);
|
||||
if (Number.isNaN(numValue)) continue;
|
||||
|
||||
points.push({
|
||||
date: new Date(`${obs.date}T00:00:00Z`),
|
||||
value: numValue,
|
||||
});
|
||||
}
|
||||
|
||||
return points;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Convenience wrappers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function formatDateParam(date?: Date | string): string | undefined {
|
||||
if (!date) return undefined;
|
||||
if (typeof date === 'string') return date;
|
||||
return date.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
export async function getNaturalGasPrice(
|
||||
startDate?: Date | string,
|
||||
endDate?: Date | string,
|
||||
): Promise<FredApiResult<CommodityPrice[]>> {
|
||||
return getCommodityPrices('natural_gas', startDate, endDate);
|
||||
}
|
||||
|
||||
export async function getWTICrudePrice(
|
||||
startDate?: Date | string,
|
||||
endDate?: Date | string,
|
||||
): Promise<FredApiResult<CommodityPrice[]>> {
|
||||
return getCommodityPrices('wti_crude', startDate, endDate);
|
||||
}
|
||||
|
||||
export async function getCoalPrice(
|
||||
startDate?: Date | string,
|
||||
endDate?: Date | string,
|
||||
): Promise<FredApiResult<CommodityPrice[]>> {
|
||||
return getCommodityPrices('coal', startDate, endDate);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared commodity fetcher
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function getCommodityPrices(
|
||||
commodity: CommodityType,
|
||||
startDate?: Date | string,
|
||||
endDate?: Date | string,
|
||||
): Promise<FredApiResult<CommodityPrice[]>> {
|
||||
const seriesId = FRED_SERIES[commodity];
|
||||
const result = await getSeriesObservations(seriesId, {
|
||||
observationStart: formatDateParam(startDate),
|
||||
observationEnd: formatDateParam(endDate),
|
||||
});
|
||||
|
||||
if (!result.ok) return result;
|
||||
|
||||
const unit = COMMODITY_UNITS[commodity];
|
||||
const prices: CommodityPrice[] = result.data.map(point => ({
|
||||
commodity,
|
||||
price: point.value,
|
||||
unit,
|
||||
timestamp: point.date,
|
||||
source: 'FRED',
|
||||
}));
|
||||
|
||||
return { ok: true, data: prices };
|
||||
}
|
||||
94
src/lib/schemas/commodities.ts
Normal file
94
src/lib/schemas/commodities.ts
Normal file
@ -0,0 +1,94 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
export const CommodityType = z.enum(['natural_gas', 'wti_crude', 'coal']);
|
||||
export type CommodityType = z.infer<typeof CommodityType>;
|
||||
|
||||
export const CommodityPriceSchema = z.object({
|
||||
commodity: CommodityType,
|
||||
price: z.number(),
|
||||
unit: z.string(),
|
||||
timestamp: z.date(),
|
||||
source: z.string(),
|
||||
});
|
||||
export type CommodityPrice = z.infer<typeof CommodityPriceSchema>;
|
||||
|
||||
export const COMMODITY_UNITS: Record<CommodityType, string> = {
|
||||
natural_gas: '$/Million BTU',
|
||||
wti_crude: '$/Barrel',
|
||||
coal: '$/Metric Ton',
|
||||
};
|
||||
|
||||
/** Coerce EIA string values to numbers, treating empty/null as null */
|
||||
const eiaNumericValue = z.union([z.string(), z.number(), z.null()]).transform((val): number | null => {
|
||||
if (val === null || val === '') return null;
|
||||
const num = Number(val);
|
||||
return Number.isNaN(num) ? null : num;
|
||||
});
|
||||
|
||||
/**
|
||||
* EIA natural gas price row.
|
||||
* Endpoint: /v2/natural-gas/pri/fut/data/
|
||||
* Series: RNGWHHD (Henry Hub Natural Gas Spot Price)
|
||||
*/
|
||||
export const eiaNaturalGasRowSchema = z.object({
|
||||
period: z.string(),
|
||||
'series-description': z.string(),
|
||||
value: eiaNumericValue,
|
||||
units: z.string().optional(),
|
||||
});
|
||||
|
||||
export type EiaNaturalGasRow = z.infer<typeof eiaNaturalGasRowSchema>;
|
||||
|
||||
/**
|
||||
* EIA petroleum/crude oil price row.
|
||||
* Endpoint: /v2/petroleum/pri/spt/data/
|
||||
* Series: RWTC (WTI Cushing Oklahoma Spot Price)
|
||||
*/
|
||||
export const eiaWtiCrudeRowSchema = z.object({
|
||||
period: z.string(),
|
||||
'series-description': z.string().optional(),
|
||||
value: eiaNumericValue,
|
||||
units: z.string().optional(),
|
||||
});
|
||||
|
||||
export type EiaWtiCrudeRow = z.infer<typeof eiaWtiCrudeRowSchema>;
|
||||
|
||||
/**
|
||||
* Parsed commodity price data point (from EIA).
|
||||
*/
|
||||
export interface CommodityPricePoint {
|
||||
timestamp: Date;
|
||||
commodity: 'natural_gas' | 'wti_crude';
|
||||
price: number | null;
|
||||
unit: string;
|
||||
source: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic EIA API response wrapper for commodity endpoints.
|
||||
*/
|
||||
export function eiaCommodityResponseSchema<T extends z.ZodType>(dataRowSchema: T) {
|
||||
return z.object({
|
||||
response: z.object({
|
||||
total: z.union([z.string(), z.number()]).transform(Number),
|
||||
data: z.array(dataRowSchema),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export const eiaNaturalGasResponseSchema = eiaCommodityResponseSchema(eiaNaturalGasRowSchema);
|
||||
export const eiaWtiCrudeResponseSchema = eiaCommodityResponseSchema(eiaWtiCrudeRowSchema);
|
||||
|
||||
export type EiaNaturalGasResponse = z.infer<typeof eiaNaturalGasResponseSchema>;
|
||||
export type EiaWtiCrudeResponse = z.infer<typeof eiaWtiCrudeResponseSchema>;
|
||||
|
||||
/**
|
||||
* Parse an EIA daily period string to a UTC Date.
|
||||
* Commodity data uses daily format: "2026-02-02"
|
||||
*/
|
||||
export function parseEiaCommodityPeriod(period: string): Date {
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(period)) {
|
||||
return new Date(`${period}T00:00:00Z`);
|
||||
}
|
||||
return new Date(period);
|
||||
}
|
||||
152
src/lib/schemas/electricity.ts
Normal file
152
src/lib/schemas/electricity.ts
Normal file
@ -0,0 +1,152 @@
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* EIA respondent codes for the 7 ISOs we track.
|
||||
* Note: EIA uses different codes than the common abbreviations.
|
||||
*/
|
||||
export const EIA_RESPONDENT_CODES = {
|
||||
PJM: 'PJM',
|
||||
ERCOT: 'ERCO',
|
||||
CAISO: 'CISO',
|
||||
NYISO: 'NYIS',
|
||||
ISONE: 'ISNE',
|
||||
MISO: 'MISO',
|
||||
SPP: 'SWPP',
|
||||
} as const;
|
||||
|
||||
export type RegionCode = keyof typeof EIA_RESPONDENT_CODES;
|
||||
export type EiaRespondentCode = (typeof EIA_RESPONDENT_CODES)[RegionCode];
|
||||
|
||||
/** Maps EIA respondent codes back to our region codes */
|
||||
export const RESPONDENT_TO_REGION: Record<EiaRespondentCode, RegionCode> = {
|
||||
PJM: 'PJM',
|
||||
ERCO: 'ERCOT',
|
||||
CISO: 'CAISO',
|
||||
NYIS: 'NYISO',
|
||||
ISNE: 'ISONE',
|
||||
MISO: 'MISO',
|
||||
SWPP: 'SPP',
|
||||
};
|
||||
|
||||
/** Type guard: check if a string is a valid EIA respondent code */
|
||||
export function isEiaRespondentCode(code: string): code is EiaRespondentCode {
|
||||
return code in RESPONDENT_TO_REGION;
|
||||
}
|
||||
|
||||
/** Safely resolve a respondent string to a RegionCode, or undefined if unknown */
|
||||
export function resolveRegionCode(respondent: string): RegionCode | undefined {
|
||||
if (isEiaRespondentCode(respondent)) {
|
||||
return RESPONDENT_TO_REGION[respondent];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Coerce EIA string values to numbers, treating empty/null as null */
|
||||
const eiaNumericValue = z.union([z.string(), z.number(), z.null()]).transform((val): number | null => {
|
||||
if (val === null || val === '') return null;
|
||||
const num = Number(val);
|
||||
return Number.isNaN(num) ? null : num;
|
||||
});
|
||||
|
||||
/**
|
||||
* EIA region-data response row.
|
||||
* Endpoint: /v2/electricity/rto/region-data/data/
|
||||
* Provides hourly demand (D) and net generation (NG) by balancing authority.
|
||||
*/
|
||||
export const eiaRegionDataRowSchema = z.object({
|
||||
period: z.string(),
|
||||
respondent: z.string(),
|
||||
'respondent-name': z.string(),
|
||||
type: z.string(),
|
||||
'type-name': z.string(),
|
||||
value: eiaNumericValue,
|
||||
'value-units': z.string(),
|
||||
});
|
||||
|
||||
export type EiaRegionDataRow = z.infer<typeof eiaRegionDataRowSchema>;
|
||||
|
||||
/**
|
||||
* EIA fuel-type-data response row.
|
||||
* Endpoint: /v2/electricity/rto/fuel-type-data/data/
|
||||
* Provides hourly generation by fuel type (e.g., NG, SUN, WND, NUC, WAT, COL).
|
||||
*/
|
||||
export const eiaFuelTypeDataRowSchema = z.object({
|
||||
period: z.string(),
|
||||
respondent: z.string(),
|
||||
'respondent-name': z.string(),
|
||||
fueltype: z.string(),
|
||||
'type-name': z.string(),
|
||||
value: eiaNumericValue,
|
||||
'value-units': z.string(),
|
||||
});
|
||||
|
||||
export type EiaFuelTypeDataRow = z.infer<typeof eiaFuelTypeDataRowSchema>;
|
||||
|
||||
/**
|
||||
* Parsed region data after transformation from the raw EIA response.
|
||||
*/
|
||||
export interface RegionDataPoint {
|
||||
timestamp: Date;
|
||||
regionCode: RegionCode;
|
||||
respondent: string;
|
||||
type: string;
|
||||
typeName: string;
|
||||
valueMw: number | null;
|
||||
valueUnits: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parsed fuel type data after transformation.
|
||||
*/
|
||||
export interface FuelTypeDataPoint {
|
||||
timestamp: Date;
|
||||
regionCode: RegionCode;
|
||||
respondent: string;
|
||||
fuelType: string;
|
||||
typeName: string;
|
||||
generationMw: number | null;
|
||||
valueUnits: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* EIA API pagination metadata.
|
||||
* Note: `total` comes back as a string from EIA.
|
||||
*/
|
||||
export const eiaPaginationSchema = z.object({
|
||||
offset: z.number(),
|
||||
length: z.number(),
|
||||
total: z.union([z.string(), z.number()]).transform(Number),
|
||||
});
|
||||
|
||||
/**
|
||||
* Generic EIA API response wrapper.
|
||||
* All EIA v2 responses share this structure.
|
||||
*/
|
||||
export function eiaResponseSchema<T extends z.ZodType>(dataRowSchema: T) {
|
||||
return z.object({
|
||||
response: z.object({
|
||||
total: z.union([z.string(), z.number()]).transform(Number),
|
||||
data: z.array(dataRowSchema),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export const eiaRegionDataResponseSchema = eiaResponseSchema(eiaRegionDataRowSchema);
|
||||
export const eiaFuelTypeDataResponseSchema = eiaResponseSchema(eiaFuelTypeDataRowSchema);
|
||||
|
||||
export type EiaRegionDataResponse = z.infer<typeof eiaRegionDataResponseSchema>;
|
||||
export type EiaFuelTypeDataResponse = z.infer<typeof eiaFuelTypeDataResponseSchema>;
|
||||
|
||||
/**
|
||||
* Parse an EIA hourly period string to a UTC Date.
|
||||
* Format: "2026-02-11T08" => Date for 2026-02-11 08:00:00 UTC
|
||||
*/
|
||||
export function parseEiaPeriod(period: string): Date {
|
||||
if (/^\d{4}-\d{2}-\d{2}T\d{2}$/.test(period)) {
|
||||
return new Date(`${period}:00:00Z`);
|
||||
}
|
||||
if (/^\d{4}-\d{2}-\d{2}$/.test(period)) {
|
||||
return new Date(`${period}T00:00:00Z`);
|
||||
}
|
||||
return new Date(period);
|
||||
}
|
||||
@ -43,7 +43,11 @@
|
||||
"paths": {
|
||||
"@/*": [
|
||||
"./src/*"
|
||||
]
|
||||
],
|
||||
"next/font": ["./node_modules/next/font/index.d.ts"],
|
||||
"next/font/local": ["./node_modules/next/font/local/index.d.ts"],
|
||||
"next/font/google": ["./node_modules/next/font/google/index.d.ts"],
|
||||
"next/*": ["./node_modules/next/*.d.ts"]
|
||||
}
|
||||
},
|
||||
"include": [
|
||||
@ -51,6 +55,7 @@
|
||||
"next.config.ts",
|
||||
"prisma.config.ts",
|
||||
"prisma/seed.ts",
|
||||
"scripts/**/*.ts",
|
||||
"src/**/*.ts",
|
||||
"src/**/*.tsx",
|
||||
".next/types/**/*.ts",
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user