import gzip
def records(frame):
return [{k: (None if isinstance(v, float) and not np.isfinite(v) else v) for k, v in row.items()}
for row in frame.to_dict(orient="records")]
field_dir = assets / "fields"
field_dir.mkdir(exist_ok=True)
for old in list(field_dir.glob("*")) + [assets / "interactive-data.json"]:
old.unlink(missing_ok=True)
grid_mask = cells.transpose("Y", "X")
flat = np.flatnonzero(grid_mask.values.ravel()) # row 0 = northernmost pixel row
n = len(flat)
def at_pixels(a):
return np.asarray(on_grid(a).transpose("Y", "X").values, dtype="float64").ravel()[flat]
def encode(values, kind, scale):
v = np.round(values * scale)
if kind == "i16":
return np.where(np.isfinite(v), np.clip(v, -32767, 32767), -32768).astype("<i2").tobytes()
if kind == "u16":
return np.where(np.isfinite(v), np.clip(v, 0, 65534), 65535).astype("<u2").tobytes()
return np.where(np.isfinite(v), np.clip(v, 0, 254), 255).astype("u1").tobytes()
def write(name, parts):
(field_dir / f"{name}.bin.gz").write_bytes(gzip.compress(b"".join(encode(*q) for q in parts), 9, mtime=0))
(assets / "pixels.bin").write_bytes(flat.astype("<i4").tobytes())
lead_key = {lead: f"lead{lead}" for lead in leads}
target_info, issue_targets = [], []
for lead in leads:
F0 = inputs["issue", lead]["F"][0]
first, last = pd.Timestamp(F0["Ti"].values[0]), pd.Timestamp(F0["Tf"].values[0])
days = (last - first).days + 1
target_info.append({"lead": lead, "label": label[lead], "pycpt": targets[lead], "start": f"{first:%Y-%m-%d}",
"end": f"{last + pd.Timedelta(days=1):%Y-%m-%d}"})
issue_targets.append({"lead": lead_key[lead], "start": f"{first:%Y-%m-%d}", "days": int(days)})
write(f"obs-{days}-{first:%Y-%m-%d}", [(at_pixels(observed[lead]["total"]), "u16", 10), (at_pixels(observed[lead]["mean"]), "u16", 10),
(at_pixels(observed[lead]["category"]), "u8", 1)])
for m in ["CCA", "PCR"]:
write(f"skill-{m}-pearson-{lead_key[lead]}", [(at_pixels(results["issue", lead, m]["skill"].pearson), "i16", 1000)])
write(f"skill-{m}-twoafc-{lead_key[lead]}", [(at_pixels(results["issue", lead, m]["skill"].two_alternative_forced_choice), "i16", 10)])
write(f"fc-{lead_key[lead]}-{m}-{first:%Y-%m-%d}", [(at_pixels(deterministic[lead, m]), "u16", 10),
(at_pixels(most_likely[lead, m][0]), "u8", 1),
(at_pixels(most_likely[lead, m][1].where(cells)), "u8", 1)])
write(f"skill-raw-pearson-{lead_key[lead]}", [(at_pixels(raw_pearson[lead]), "i16", 1000)])
values = np.concatenate([observed[lead]["total"].values.ravel() for lead in leads] + [deterministic[k].values.ravel() for k in deterministic])
top = float(np.ceil(np.nanpercentile(values, 99) / 100) * 100)
manifest = {
"grid": {"west": round(float(grid_mask.X.min()) - .025, 3), "north": round(float(grid_mask.Y.max()) + .025, 3), "step": 0.05,
"nx": int(grid_mask.sizes["X"]), "ny": int(grid_mask.sizes["Y"]), "pixels": n},
"kind": "month", "issue": {"date": issue.date().isoformat(), "label": "1 Nov 2023"},
"issues": [{"issued": issue.date().isoformat(), "label": "1 Nov 2023", "targets": issue_targets}],
"leads": leads, "leadKeys": [lead_key[lead] for lead in leads], "methods": ["CCA", "PCR"], "rawName": "raw CFSv2", "unit": "mm",
"training": "1991-2020", "sample": "the 30 training years, 1991-2020", "cellText": f"Indonesia's {n:,} pixels at 0.05°",
"leadNames": [f"Lead {lead} · {label[lead]}" for lead in leads],
"targets": target_info,
"layout": {"skill": [["value", "i16"]], "obs": [["total", "u16", 10], ["mean", "u16", 10], ["category", "u8", 1]],
"fc": [["total", "u16", 10], ["category", "u8", 1], ["probability", "u8", 1]]},
"scales": {"pearson": 1000, "twoafc": 10},
"rainMax": {str(t["days"]): top for t in issue_targets}, "rainLegend": "mm in the month",
"mapsHint": "Lead 1 at the top. Each map is a different target; hover a pixel for the forecast, the observation and that target's 1991-2020 mean.",
"skillTable": records(skill_by_lead.reset_index().round(4)),
"scores2023": records(forecast_2023.reset_index().round(3)),
"references": records(targets_2023.reset_index().round(3)),
"sameTarget": {"target": "November 2023", "rows": records(same_target.reset_index().round(4))},
}
if "comparison" in globals():
manifest["comparison"] = records(comparison.reset_index().round(4))
(assets / "manifest.json").write_text(json.dumps(manifest, separators=(",", ":"), allow_nan=False) + "\n")
sizes = sorted(q.stat().st_size for q in field_dir.iterdir())
print(f"{n:,} pixels; {len(sizes)} field files, {sum(sizes) / 1e6:.1f} MB in total (largest {sizes[-1] / 1e3:.0f} kB); "
f"colour scale up to {top:.0f} mm")
