{ "archive_built_utc": "2026-08-18T05:18:57Z", "archive_stamp": "7fa8f68e14bda1e8", "counts": { "environment": 33324, "genesis_events": 3959, "landfalls": 3379, "storms": 3959, "track_points": 224153 }, "empty_in_archive": { "_note": "These columns exist in scripts/genesis/schema.py and carry no data in this archive. The per-storm pre-genesis linkage they imply has never been merged; the linkage that does exist is daily_disturbances.resolved_storm_id, which this pack does not carry.", "environment.lead_hours": "0.0 on every row -- analysis time only, no forecast leads", "genesis_events.invest_id": "NULL on every row", "genesis_events.pregenesis_source": "'none' on every row", "genesis_events.two_first_lat": "NULL on every row", "genesis_events.two_first_lon": "NULL on every row", "genesis_events.two_first_mention_utc": "NULL on every row", "genesis_events.two_lead_hours": "NULL on every row" }, "env_coverage": { "by_source": { "ships_dev": { "first_utc": "1982-05-20 00:00:00+00:00", "last_utc": "2022-11-11 12:00:00+00:00", "rows": 32003, "storms": 1408 }, "ships_dev+csst": { "first_utc": "2023-01-16 12:00:00+00:00", "last_utc": "2023-11-26 12:00:00+00:00", "rows": 839, "storms": 41 }, "ships_rt": { "first_utc": "2026-03-11 18:00:00+00:00", "last_utc": "2026-09-24 12:00:00+00:00", "rows": 482, "storms": 12 } }, "note": "Sources are reported separately and must not be pooled: ships_dev+csst carries a CLIMATOLOGICAL sea-surface temperature where ships_dev carries an observed one. No storm in this archive draws on more than one source, but a matched pool can contain storms from several.", "storms_any_source": 1461, "storms_total": 3959 }, "env_genesis_window_hours": 12.0, "files": { "atlas-core-v1.bin.gz": { "bytes": 431800, "raw_bytes": 1524928, "sha256": "bc314f2510a078f743a2db5551d7904a97ec25597068801c9f809474d5baaee5" }, "atlas-env-v1.bin.gz": { "bytes": 1004945, "raw_bytes": 5309640, "sha256": "82c24dd365deeba6f2bf9869264fdf5bd48bc374ce54786b7f712533f0ab3aba" }, "atlas-tracks-v1.bin.gz": { "bytes": 971885, "raw_bytes": 4067766, "sha256": "6d91b08907321418151a21ccce775177baf17daf892d7a1acb1e09aca87abfab" } }, "methodology_version": "1.1.1", "not_packed": { "_note": "Absent from the pack, present in the archive. Nothing here was dropped because it was inconvenient; each is unread by any surface this pack serves.", "daily_disturbances": "whole table -- pre-genesis evidence is a later phase", "track_points": [ "storm_speed_kt", "storm_dir_deg", "dist2land_km", "hours_since_genesis (exactly derivable)" ] }, "pack_format": 1, "processing_version": "1.0.0", "provenance": { "archive_built_utc": "2026-08-18T05:18:57Z", "archive_dir": "data/genesis-archive", "archive_sources": [ "ibtracs.EP.csv", "ibtracs.NA.csv", "ships.EP.txt", "ships.CP.txt", "ships.AL.txt", "hurdat2.nepac.txt", "hurdat2.atl.txt" ], "archive_stamp": "7fa8f68e14bda1e8", "cohort_archive_id": "de6b263809e7a7af3d5251020b7b04ec723a22150015b2732fa17ee2fa9525c1", "cohort_archive_id_derivation": "cohort-archive-id/1", "cohort_archive_id_tables": [ "genesis_events", "landfalls", "storms", "track_points" ], "gaps": [ { "attempted_utc": "2026-08-18T05:18:29Z", "impact": "those environment rows have storm_id NULL and cannot be analog-matched", "key": "ships_join", "url": "", "what": "SHIPS records with no IBTrACS storm", "why": "167 SHIPS records carry an ATCF id absent from the loaded IBTrACS basins (a basin not loaded, or a storm IBTrACS did not adopt)" }, { "attempted_utc": "2026-08-18T05:18:29Z", "impact": "Every environment row this module emits is 2.5 degrees and 6-hourly instead of ERA5's 0.25 degrees and hourly -- roughly 100x coarser in area and 6x in time. A TC inner core is not resolved at all, the 'environment' is an average over a ~275 km cell, and the sampled cell can be up to ~140 km from the position requested. Genesis-scale features (a mesoscale vortex inside a tropical wave) are below the grid and simply absent. Do not compare these fields quantitatively with any ERA5-derived study.", "key": "era5", "url": "https://cds.climate.copernicus.eu/", "what": "ERA5 is not in this archive at all; NCEP/NCAR Reanalysis 1 was substituted, and it is a coarser instrument", "why": "Both published routes to ERA5 were tried from this environment and both refused. (1) Copernicus CDS: POST to https://cds.climate.copernicus.eu/api/retrieve/v1/processes/reanalysis-era5-pressure-levels/execution returns HTTP 401 {\"detail\":\"authentication required\"}; the API needs a personal access token and none is provisioned here (no ~/.cdsapirc, no CDSAPI_* in the environment). The public process listing returns HTTP 200 from the same host, so the network path is open and the missing thing is a credential. (2) The AWS Open Data mirror era5-pds returns HTTP 403 AccessDenied for both an object GET and a bucket listing." }, { "attempted_utc": "2026-08-18T05:18:29Z", "impact": "rh_mid_pct is read at 600 hPa, the published level nearest the middle of the SHIPS 700-500 hPa layer, and it is a SINGLE LEVEL, not a layer mean and not a storm-relative annulus average -- it is not the same quantity as the ships_dev rows in the same column. Upper-level (250/200 hPa) moisture cannot be obtained from this source at all.", "key": "ncep_r1", "url": "https://psl.noaa.gov/thredds/dodsC/Datasets/ncep.reanalysis/pressure/rhum.2020.nc.dds", "what": "rhum is published on FEWER levels than the winds: 8, not 17", "why": "Verified against the live files for 2020: uwnd/vwnd/air/hgt carry level[17] = 1000 925 850 700 600 500 400 300 250 200 150 100 70 50 30 20 10, while rhum and shum carry only level[8] = 1000 925 850 700 600 500 400 300. There is no relative humidity above 300 hPa in this dataset." }, { "attempted_utc": "2026-08-18T05:18:29Z", "impact": "These are environmental (monsoon-trough / ITCZ) values. A SHIPS Z850 is a 0-1000 km storm-relative average that retains the vortex, so the two are not interchangeable and pooling them across env_source would mix scales. Points on the first or last grid row get NULL rather than a one-sided difference.", "key": "ncep_r1", "url": "", "what": "vort850_1e5 and div200_1e7 are ~550 km centred differences, not storm-scale fields", "why": "The grid is 2.5 degrees, so the tightest centred difference available spans two cells: 5 degrees of longitude (~545 km at 12 N) and 5 degrees of latitude (~556 km). The spherical form used is zeta = (1/(a cos phi)) [dv/dlam - d(u cos phi)/dphi]." }, { "attempted_utc": "2026-08-18T05:18:29Z", "impact": "Match ncep_r1 rows against ncep_r1 rows. Pooling env_source across a humidity comparison pools two columns that share only about 3 percent of their variance (r=+0.16); for shear, vorticity, divergence and temperature it pools related numbers carrying a scale offset; only u200_kt is close to interchangeable. These are magnitudes from n=150, not a calibration: at n=30 the humidity correlation alone ranges from -0.40 to +0.61 across random subsets, which is why no correction factor is published here -- fitting one and applying it to the archive would be manufacturing agreement the sources do not have.", "key": "ncep_r1", "url": "", "what": "these columns are NOT drop-in replacements for the ships_dev columns of the same name, and the difference was MEASURED, not assumed", "why": "150 SHIPS CP developmental records at tau=0, drawn at random from the 996 in the cached file, were run through environment_at() at the same lat, lon and time and compared column by column. Every difference below is stated in ONE direction, ncep_r1 minus ships_dev, as a median of the paired differences (which is not the difference of the two medians). shear_kt: SHIPS median 11.7 kt, ncep_r1 15.0 kt, median difference +4.4 kt, Pearson r=+0.66. vort850_1e5: 0.40 vs 2.07, +1.64, r=+0.49. rh_mid_pct: 55.5 vs 49.5 percent, -8.0, r=+0.16. u200_kt: -2.0 vs -2.1, +0.2, r=+0.88 -- the one column where the two sources nearly agree, because U200 is a plain gridded single-level wind in both. t200_c: -53.4 vs -51.8, +1.6, r=+0.66. tpw_mm: 59.8 vs 46.2, -11.9, r=+0.39, the SHIPS column being a 0-200 km storm-centred average and this one a single 2.5 degree cell. div200_1e7: 36.5 vs 53.7, +18.8, r=+0.47. Humidity is the weakest link of the set and the reason is structural: a 700-500 hPa layer average over a 200-800 km storm-relative annulus is not a single 600 hPa level in one 2.5 degree cell." }, { "attempted_utc": "2026-08-18T05:18:29Z", "impact": "A query that pools mslp_env_mb across env_source is comparing ambient pressure with storm intensity. Filter on env_source.", "key": "ncep_r1", "url": "", "what": "mslp_env_mb means something different here than it does on ships_dev rows", "why": "Here it is the NCEP R1 grid-cell sea level pressure (DAS units 'Pascals', divided by 100), i.e. a genuine ambient pressure. On ships_dev rows the same column carries SHIPS MSLP, which that module documents as the storm's own minimum central pressure." }, { "attempted_utc": "2026-08-18T05:18:36Z", "impact": "those landfalls are absent from the archive", "key": "landfall_join", "url": "", "what": "9 HURDAT2 landfalls unjoined to IBTrACS", "why": "the storm's ATCF id is not present in the loaded IBTrACS basins" } ], "table_sha256": { "daily_disturbances": { "bytes": 352998, "sha256": "578512d62eeb60c10feeae92b559dabab192df25f7decde61d49b5f28627453b" }, "environment": { "bytes": 1035588, "sha256": "15b62f9e957d9ff489404b27fc886ab8d5376b9b98fd7b881173900a3554f155" }, "genesis_events": { "bytes": 239460, "sha256": "5fe9f13d72114e5f1227ead7d529a1443aabbf2335e3b61fc6915f9896a632cf" }, "landfalls": { "bytes": 130427, "sha256": "5cfd2f722dbf375cd873cc39c0f000dcb1d689327989fea7d60ce1b19cca9f46" }, "storms": { "bytes": 135795, "sha256": "ec45a23578d2400ab891d1a7174649b6ff1c6a4aa90df20b9628a7ac6e716689" }, "track_points": { "bytes": 2658123, "sha256": "865d905326b1e2137867ddd0bf0159cf45599a9779ab8dbc79526546561c0b76" } } }, "quality": { "env_source": { "ships_dev": 32003, "ships_dev+csst": 839, "ships_rt": 482 }, "landfall_category_withheld": 466, "landfall_detection": { "bracketing_fix": 1061, "hurdat2_L_record": 1305, "segment_crossing": 1013 }, "storms_interpolated_only": 26, "storms_with_env_at_genesis": 1431, "storms_with_genesis": 3905, "storms_with_no_track_points": 0, "track_points": { "interpolated": 109171, "observed": 114316, "provisional": 666 } }, "subbasin_bits": [ "NA", "EP", "CP", "CS", "GM", "AS", "WP", "NI" ], "thresholds_kt": { "cat1": 64, "cat2": 83, "cat3": 96, "cat4": 113, "cat5": 137, "td": 0, "ts": 34 }, "track_geometry": { "max_deviation_deg": 2.842170943040401e-14, "note": "Track positions are carried as int16 hundredths of a degree, which is IBTrACS's own published precision. The deviation above is the worst difference between an archive value and its packed form; it arises from the archive's signed-longitude arithmetic, not from a loss of source precision. Genesis, threshold-crossing and landfall coordinates -- the ones the query engine computes distances from -- are carried as float64 and are not quantised at all.", "quantised_to_deg": 0.01 } }