# Files changed · PR \#119 · flambo/atlantis [View on GitCafe](https://git.cafe/flambo/atlantis/pull/119/diffs) Repository: [flambo/atlantis](https://git.cafe/flambo/atlantis) Visibility: public [feat: store MAAD results as compact f32 rows](https://git.cafe/flambo/atlantis/pull/119.md) Version: 0 Head commit: 08c62c11f76e3a1c9ea8033f108b7209bbe786f5 Base commit: da1fe86c996136d7df3ccb81325523159a618144 Comparison base: da1fe86c996136d7df3ccb81325523159a618144 Showing up to 20 entries from this revision. If the pull request changes, reopen the first page before continuing. ## apps/web/drizzle/0006_compact_maad_storage.sql Status: added; +48; −0 ```diff @@ -1,0 +1,48 @@ +CREATE TABLE `address_maad_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `address_side` text NOT NULL, + `measure` text NOT NULL, + `total_addrs` integer NOT NULL, + `zero_weight_addrs` integer DEFAULT 0 NOT NULL, + `min_prefix_length` integer, + `max_prefix_length` integer, + `d0` real, + `d1` real, + `d2` real, + `tau` blob, + `tau_sd` blob, + `spectrum` blob, + CONSTRAINT "address_maad_stats_ip_version_check" CHECK("address_maad_stats"."ip_version" IN (4, 6)), + CONSTRAINT "address_maad_stats_src_locality_check" CHECK("address_maad_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_maad_stats_dst_locality_check" CHECK("address_maad_stats"."dst_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_maad_stats_measure_check" CHECK("address_maad_stats"."measure" IN ('addresses', 'packets', 'bytes')), + CONSTRAINT "address_maad_stats_bucket_check" CHECK("address_maad_stats"."bucket_end" > "address_maad_stats"."bucket_start"), + CONSTRAINT "address_maad_stats_total_addrs_check" CHECK("address_maad_stats"."total_addrs" >= 0), + CONSTRAINT "address_maad_stats_zero_weight_addrs_check" CHECK("address_maad_stats"."zero_weight_addrs" >= 0 AND ("address_maad_stats"."measure" <> 'addresses' OR "address_maad_stats"."zero_weight_addrs" = 0)), + CONSTRAINT "address_maad_stats_prefix_length_check" CHECK("address_maad_stats"."max_prefix_length" >= "address_maad_stats"."min_prefix_length"), + CONSTRAINT "address_maad_stats_tau_check" CHECK(typeof("address_maad_stats"."tau") IN ('null', 'blob') AND length("address_maad_stats"."tau") > 0 AND length("address_maad_stats"."tau") % 4 = 0), + CONSTRAINT "address_maad_stats_tau_sd_check" CHECK(typeof("address_maad_stats"."tau_sd") = typeof("address_maad_stats"."tau") AND length("address_maad_stats"."tau_sd") IS length("address_maad_stats"."tau")), + CONSTRAINT "address_maad_stats_spectrum_check" CHECK(typeof("address_maad_stats"."spectrum") IN ('null', 'blob') AND length("address_maad_stats"."spectrum") % 8 = 0), + CONSTRAINT "address_maad_stats_measure_spectrum_check" CHECK("address_maad_stats"."measure" = 'addresses' OR "address_maad_stats"."spectrum" IS NULL), + CONSTRAINT "address_maad_stats_tau_d0_check" CHECK(("address_maad_stats"."tau" IS NULL) = ("address_maad_stats"."d0" IS NULL)) +); +--> statement-breakpoint +CREATE UNIQUE INDEX `idx_address_maad_stats_key` ON `address_maad_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`ip_version`,`measure`,`bucket_start`,`address_side`);--> statement-breakpoint +CREATE INDEX `idx_address_maad_stats_bucket` ON `address_maad_stats` (`granularity`,`bucket_start`);--> statement-breakpoint +CREATE TABLE `maad_q_grid` ( + `ip_version` integer PRIMARY KEY NOT NULL, + `q_min` real NOT NULL, + `q_step` real NOT NULL, + `q_count` integer NOT NULL, + CONSTRAINT "maad_q_grid_ip_version_check" CHECK("maad_q_grid"."ip_version" IN (4, 6)), + CONSTRAINT "maad_q_grid_q_step_check" CHECK("maad_q_grid"."q_step" > 0), + CONSTRAINT "maad_q_grid_q_count_check" CHECK("maad_q_grid"."q_count" > 0) +); +--> statement-breakpoint +DROP TABLE `address_structure_stats`; ``` ## apps/web/drizzle/meta/0006_snapshot.json Status: added; +1206; −0 ```diff @@ -1,0 +1,1206 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "839fdf34-c740-42cc-acd4-da4c91124f3f", + "prevId": "346de28f-0cfb-4986-8daf-28235ade1fdf", + "tables": { + "address_count_stats": { + "name": "address_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_address_count": { + "name": "unique_address_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_count_stats_query": { + "name": "idx_address_count_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "isUnique": false + }, + "idx_address_count_stats_timeseries": { + "name": "idx_address_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "name": "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_count_stats_ip_version_check": { + "name": "address_count_stats_ip_version_check", + "value": "\"address_count_stats\".\"ip_version\" IN (4, 6)" + }, + "address_count_stats_src_locality_check": { + "name": "address_count_stats_src_locality_check", + "value": "\"address_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_count_stats_dst_locality_check": { + "name": "address_count_stats_dst_locality_check", + "value": "\"address_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "address_maad_stats": { + "name": "address_maad_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "measure": { + "name": "measure", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_addrs": { + "name": "total_addrs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "zero_weight_addrs": { + "name": "zero_weight_addrs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "min_prefix_length": { + "name": "min_prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_prefix_length": { + "name": "max_prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "d0": { + "name": "d0", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "d1": { + "name": "d1", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "d2": { + "name": "d2", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tau": { + "name": "tau", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tau_sd": { + "name": "tau_sd", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spectrum": { + "name": "spectrum", + "type": "blob", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_address_maad_stats_key": { + "name": "idx_address_maad_stats_key", + "columns": [ + "source_id", + "granularity", + "src_locality", + "dst_locality", + "ip_version", + "measure", + "bucket_start", + "address_side" + ], + "isUnique": true + }, + "idx_address_maad_stats_bucket": { + "name": "idx_address_maad_stats_bucket", + "columns": ["granularity", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "address_maad_stats_ip_version_check": { + "name": "address_maad_stats_ip_version_check", + "value": "\"address_maad_stats\".\"ip_version\" IN (4, 6)" + }, + "address_maad_stats_src_locality_check": { + "name": "address_maad_stats_src_locality_check", + "value": "\"address_maad_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_maad_stats_dst_locality_check": { + "name": "address_maad_stats_dst_locality_check", + "value": "\"address_maad_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + }, + "address_maad_stats_measure_check": { + "name": "address_maad_stats_measure_check", + "value": "\"address_maad_stats\".\"measure\" IN ('addresses', 'packets', 'bytes')" + }, + "address_maad_stats_bucket_check": { + "name": "address_maad_stats_bucket_check", + "value": "\"address_maad_stats\".\"bucket_end\" > \"address_maad_stats\".\"bucket_start\"" + }, + "address_maad_stats_total_addrs_check": { + "name": "address_maad_stats_total_addrs_check", + "value": "\"address_maad_stats\".\"total_addrs\" >= 0" + }, + "address_maad_stats_zero_weight_addrs_check": { + "name": "address_maad_stats_zero_weight_addrs_check", + "value": "\"address_maad_stats\".\"zero_weight_addrs\" >= 0 AND (\"address_maad_stats\".\"measure\" <> 'addresses' OR \"address_maad_stats\".\"zero_weight_addrs\" = 0)" + }, + "address_maad_stats_prefix_length_check": { + "name": "address_maad_stats_prefix_length_check", + "value": "\"address_maad_stats\".\"max_prefix_length\" >= \"address_maad_stats\".\"min_prefix_length\"" + }, + "address_maad_stats_tau_check": { + "name": "address_maad_stats_tau_check", + "value": "typeof(\"address_maad_stats\".\"tau\") IN ('null', 'blob') AND length(\"address_maad_stats\".\"tau\") > 0 AND length(\"address_maad_stats\".\"tau\") % 4 = 0" + }, + "address_maad_stats_tau_sd_check": { + "name": "address_maad_stats_tau_sd_check", + "value": "typeof(\"address_maad_stats\".\"tau_sd\") = typeof(\"address_maad_stats\".\"tau\") AND length(\"address_maad_stats\".\"tau_sd\") IS length(\"address_maad_stats\".\"tau\")" + }, + "address_maad_stats_spectrum_check": { + "name": "address_maad_stats_spectrum_check", + "value": "typeof(\"address_maad_stats\".\"spectrum\") IN ('null', 'blob') AND length(\"address_maad_stats\".\"spectrum\") % 8 = 0" + }, + "address_maad_stats_measure_spectrum_check": { + "name": "address_maad_stats_measure_spectrum_check", + "value": "\"address_maad_stats\".\"measure\" = 'addresses' OR \"address_maad_stats\".\"spectrum\" IS NULL" + }, + "address_maad_stats_tau_d0_check": { + "name": "address_maad_stats_tau_d0_check", + "value": "(\"address_maad_stats\".\"tau\" IS NULL) = (\"address_maad_stats\".\"d0\" IS NULL)" + } + } + }, + "bucket_coverage": { + "name": "bucket_coverage", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "coverage_state": { + "name": "coverage_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "observed_units": { + "name": "observed_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expected_units": { + "name": "expected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rejected_units": { + "name": "rejected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_bucket_coverage_query": { + "name": "idx_bucket_coverage_query", + "columns": ["granularity", "bucket_start", "source_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bucket_coverage_source_id_granularity_bucket_start_pk": { + "columns": ["source_id", "granularity", "bucket_start"], + "name": "bucket_coverage_source_id_granularity_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "bucket_coverage_interval_check": { + "name": "bucket_coverage_interval_check", + "value": "\"bucket_coverage\".\"bucket_end\" > \"bucket_coverage\".\"bucket_start\"" + }, + "bucket_coverage_expected_check": { + "name": "bucket_coverage_expected_check", + "value": "\"bucket_coverage\".\"expected_units\" > 0" + }, + "bucket_coverage_observed_check": { + "name": "bucket_coverage_observed_check", + "value": "\"bucket_coverage\".\"observed_units\" >= 0 AND \"bucket_coverage\".\"observed_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_rejected_check": { + "name": "bucket_coverage_rejected_check", + "value": "\"bucket_coverage\".\"rejected_units\" >= 0 AND \"bucket_coverage\".\"rejected_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_state_check": { + "name": "bucket_coverage_state_check", + "value": "(\"bucket_coverage\".\"coverage_state\" = 'complete' AND \"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'unknown' AND \"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'partial' AND NOT (\"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) AND NOT (\"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0))" + } + } + }, + "datasets": { + "name": "datasets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_start_date": { + "name": "default_start_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_mode": { + "name": "source_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "discovery_mode": { + "name": "discovery_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_locality": { + "name": "has_locality", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "maad_q_grid": { + "name": "maad_q_grid", + "columns": { + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "q_min": { + "name": "q_min", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "q_step": { + "name": "q_step", + "type": "real", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "q_count": { + "name": "q_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "maad_q_grid_ip_version_check": { + "name": "maad_q_grid_ip_version_check", + "value": "\"maad_q_grid\".\"ip_version\" IN (4, 6)" + }, + "maad_q_grid_q_step_check": { + "name": "maad_q_grid_q_step_check", + "value": "\"maad_q_grid\".\"q_step\" > 0" + }, + "maad_q_grid_q_count_check": { + "name": "maad_q_grid_q_count_check", + "value": "\"maad_q_grid\".\"q_count\" > 0" + } + } + }, + "port_count_stats": { + "name": "port_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_side": { + "name": "port_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_range": { + "name": "port_range", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_port_count": { + "name": "unique_port_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_port_count_stats_timeseries": { + "name": "idx_port_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "port_side", + "port_range" + ], + "name": "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "port_count_stats_ip_version_check": { + "name": "port_count_stats_ip_version_check", + "value": "\"port_count_stats\".\"ip_version\" IN (4, 6)" + }, + "port_count_stats_src_locality_check": { + "name": "port_count_stats_src_locality_check", + "value": "\"port_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "port_count_stats_dst_locality_check": { + "name": "port_count_stats_dst_locality_check", + "value": "\"port_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "processed_inputs": { + "name": "processed_inputs", + "columns": { + "input_kind": { + "name": "input_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_locator": { + "name": "input_locator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "discovered_at": { + "name": "discovered_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_processed_inputs_source_bucket": { + "name": "idx_processed_inputs_source_bucket", + "columns": ["source_id", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk": { + "columns": ["input_kind", "input_locator", "source_id", "bucket_start"], + "name": "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "protocol_stats": { + "name": "protocol_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_protocols_count": { + "name": "unique_protocols_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocols_list": { + "name": "protocols_list", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_protocol_stats_timeseries": { + "name": "idx_protocol_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "protocol_stats_ip_version_check": { + "name": "protocol_stats_ip_version_check", + "value": "\"protocol_stats\".\"ip_version\" IN (4, 6)" + }, + "protocol_stats_src_locality_check": { + "name": "protocol_stats_src_locality_check", + "value": "\"protocol_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "protocol_stats_dst_locality_check": { + "name": "protocol_stats_dst_locality_check", + "value": "\"protocol_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "source_members": { + "name": "source_members", + "columns": { + "dataset_id": { + "name": "dataset_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "source_members_dataset_id_source_id_member_id_pk": { + "columns": ["dataset_id", "source_id", "member_id"], + "name": "source_members_dataset_id_source_id_member_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "traffic_stats": { + "name": "traffic_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows": { + "name": "flows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_tcp": { + "name": "flows_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_udp": { + "name": "flows_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_icmp": { + "name": "flows_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_other": { + "name": "flows_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets": { + "name": "packets", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_tcp": { + "name": "packets_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_udp": { + "name": "packets_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_icmp": { + "name": "packets_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_other": { + "name": "packets_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_tcp": { + "name": "bytes_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_udp": { + "name": "bytes_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_icmp": { + "name": "bytes_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_other": { + "name": "bytes_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_sum_ms": { + "name": "duration_sum_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_count": { + "name": "duration_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_duration_ms": { + "name": "average_duration_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_ttl_sum": { + "name": "min_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_ttl_count": { + "name": "min_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_min_ttl": { + "name": "average_min_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_ttl_sum": { + "name": "max_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_ttl_count": { + "name": "max_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_max_ttl": { + "name": "average_max_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_traffic_stats_query": { + "name": "idx_traffic_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality" + ], + "isUnique": false + }, + "idx_traffic_stats_timeseries": { + "name": "idx_traffic_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "traffic_stats_ip_version_check": { + "name": "traffic_stats_ip_version_check", + "value": "\"traffic_stats\".\"ip_version\" IN (4, 6)" + }, + "traffic_stats_src_locality_check": { + "name": "traffic_stats_src_locality_check", + "value": "\"traffic_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "traffic_stats_dst_locality_check": { + "name": "traffic_stats_dst_locality_check", + "value": "\"traffic_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} ``` ## apps/web/drizzle/meta/\_journal.json Status: modified; +7; −0 ```diff @@ -43,6 +43,13 @@ "when": 1790328400092, "tag": "0005_maad_measures", "breakpoints": true + }, + { + "idx": 6, + "version": "6", + "when": 1790331619750, + "tag": "0006_compact_maad_storage", + "breakpoints": true } ] } ``` ## apps/web/src/lib/server/datasets.ts Status: modified; +1; −1 ```diff @@ -73,7 +73,7 @@ 'traffic_stats', 'protocol_stats', 'address_count_stats', - 'address_structure_stats', + 'address_maad_stats', 'port_count_stats' ]; const currentProductSchemaSql = `SELECT ( ``` ## apps/web/src/lib/server/db/local-schema.ts Status: modified; +31; −25 ```diff @@ -112,27 +112,36 @@ PRIMARY KEY(source_id, granularity, bucket_start, ip_version, src_locality, dst_locality, address_side) ); - CREATE TABLE IF NOT EXISTS address_structure_stats ( + CREATE TABLE IF NOT EXISTS address_maad_stats ( source_id TEXT NOT NULL, - granularity TEXT NOT NULL CHECK(granularity IN ('5m', '10m', '30m', '1h', '1d')), + granularity TEXT NOT NULL CHECK (granularity IN ('5m', '10m', '30m', '1h', '1d')), bucket_start INTEGER NOT NULL, - bucket_end INTEGER NOT NULL, - ip_version INTEGER NOT NULL CHECK(ip_version IN (4, 6)), - src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), - dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), - address_side TEXT NOT NULL CHECK(address_side IN ('source', 'destination')), - measure TEXT NOT NULL CHECK(measure IN ('addresses', 'packets', 'bytes')), - structure_kind TEXT NOT NULL CHECK(structure_kind IN ('structure', 'spectrum', 'dimension')), - values_json TEXT NOT NULL, - metadata_json TEXT NOT NULL, - processed_at TEXT DEFAULT CURRENT_TIMESTAMP, - CHECK(measure = 'addresses' OR structure_kind <> 'spectrum'), - PRIMARY KEY( + bucket_end INTEGER NOT NULL CHECK (bucket_end > bucket_start), + ip_version INTEGER NOT NULL CHECK (ip_version IN (4, 6)), - source_id, granularity, bucket_start, ip_version, - src_locality, dst_locality, address_side, measure, structure_kind - ) + src_locality TEXT NOT NULL CHECK (src_locality IN ('all', 'internal', 'external')), + dst_locality TEXT NOT NULL CHECK (dst_locality IN ('all', 'internal', 'external')), + address_side TEXT NOT NULL CHECK (address_side IN ('source', 'destination')), + measure TEXT NOT NULL CHECK (measure IN ('addresses', 'packets', 'bytes')), + total_addrs INTEGER NOT NULL CHECK (total_addrs >= 0), + zero_weight_addrs INTEGER NOT NULL DEFAULT 0 + CHECK (zero_weight_addrs >= 0 AND (measure <> 'addresses' OR zero_weight_addrs = 0)), + min_prefix_length INTEGER, + max_prefix_length INTEGER CHECK (max_prefix_length >= min_prefix_length), + d0 REAL, d1 REAL, d2 REAL, + tau BLOB CHECK (typeof(tau) IN ('null', 'blob') AND length(tau) > 0 AND length(tau) % 4 = 0), + tau_sd BLOB CHECK (typeof(tau_sd) = typeof(tau) AND length(tau_sd) IS length(tau)), + spectrum BLOB CHECK (typeof(spectrum) IN ('null', 'blob') AND length(spectrum) % 8 = 0), + CHECK (measure = 'addresses' OR spectrum IS NULL), + CHECK ((tau IS NULL) = (d0 IS NULL)) ); + CREATE TABLE IF NOT EXISTS maad_q_grid ( + ip_version INTEGER PRIMARY KEY CHECK (ip_version IN (4, 6)), + q_min REAL NOT NULL, + q_step REAL NOT NULL CHECK (q_step > 0), + q_count INTEGER NOT NULL CHECK (q_count > 0) + ); + CREATE TABLE IF NOT EXISTS port_count_stats ( source_id TEXT NOT NULL, granularity TEXT NOT NULL CHECK(granularity IN ('5m', '10m', '30m', '1h', '1d')), @@ -180,16 +189,13 @@ source_id, granularity, src_locality, dst_locality, bucket_start ); - CREATE INDEX IF NOT EXISTS idx_address_structure_stats_query - ON address_structure_stats ( - granularity, bucket_start, source_id, ip_version, - src_locality, dst_locality, address_side, measure, structure_kind - ); - CREATE INDEX IF NOT EXISTS idx_address_structure_stats_timeseries - ON address_structure_stats ( + CREATE UNIQUE INDEX IF NOT EXISTS idx_address_maad_stats_key + ON address_maad_stats ( source_id, granularity, src_locality, dst_locality, + ip_version, measure, bucket_start, address_side - ip_version, measure, structure_kind, bucket_start ); + CREATE INDEX IF NOT EXISTS idx_address_maad_stats_bucket + ON address_maad_stats (granularity, bucket_start); CREATE INDEX IF NOT EXISTS idx_port_count_stats_timeseries ON port_count_stats ( source_id, granularity, src_locality, dst_locality, ``` ## apps/web/src/lib/server/db/schema.ts Status: modified; +64; −43 ```diff @@ -1,12 +1,14 @@ import { sql } from 'drizzle-orm'; import { + blob, check, index, integer, primaryKey, real, sqliteTable, - text + text, + uniqueIndex } from 'drizzle-orm/sqlite-core'; const currentTimestamp = sql`CURRENT_TIMESTAMP`; @@ -320,8 +322,8 @@ ] ); -export const addressStructureStats = sqliteTable( +export const addressMaadStats = sqliteTable( - 'address_structure_stats', + 'address_maad_stats', { sourceId: text('source_id').notNull(), granularity: text('granularity', { enum: ['5m', '10m', '30m', '1h', '1d'] }).notNull(), @@ -332,64 +334,83 @@ dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), addressSide: text('address_side', { enum: ['source', 'destination'] }).notNull(), measure: text('measure', { enum: ['addresses', 'packets', 'bytes'] }).notNull(), + totalAddrs: integer('total_addrs').notNull(), + zeroWeightAddrs: integer('zero_weight_addrs').notNull().default(0), + minPrefixLength: integer('min_prefix_length'), + maxPrefixLength: integer('max_prefix_length'), + d0: real('d0'), + d1: real('d1'), + d2: real('d2'), + tau: blob('tau', { mode: 'buffer' }), + tauSd: blob('tau_sd', { mode: 'buffer' }), + spectrum: blob('spectrum', { mode: 'buffer' }) - structureKind: text('structure_kind', { - enum: ['structure', 'spectrum', 'dimension'] - }).notNull(), - valuesJson: text('values_json').notNull(), - metadataJson: text('metadata_json').notNull(), - processedAt: text('processed_at').default(currentTimestamp) }, (table) => [ - primaryKey({ - columns: [ - table.sourceId, - table.granularity, - table.bucketStart, - table.ipVersion, - table.srcLocality, - table.dstLocality, - table.addressSide, - table.measure, - table.structureKind - ] - }), + uniqueIndex('idx_address_maad_stats_key').on( - index('idx_address_structure_stats_query').on( - table.granularity, - table.bucketStart, table.sourceId, - table.ipVersion, - table.srcLocality, - table.dstLocality, - table.addressSide, - table.measure, - table.structureKind - ), - index('idx_address_structure_stats_timeseries').on( - table.sourceId, table.granularity, table.srcLocality, table.dstLocality, table.ipVersion, table.measure, - table.structureKind, + table.bucketStart, + table.addressSide - table.bucketStart ), - check('address_structure_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), + index('idx_address_maad_stats_bucket').on(table.granularity, table.bucketStart), + check('address_maad_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), check( + 'address_maad_stats_src_locality_check', - 'address_structure_stats_src_locality_check', sql`${table.srcLocality} IN ('all', 'internal', 'external')` ), check( + 'address_maad_stats_dst_locality_check', - 'address_structure_stats_dst_locality_check', sql`${table.dstLocality} IN ('all', 'internal', 'external')` ), check( - 'address_structure_stats_measure_check', + 'address_maad_stats_measure_check', sql`${table.measure} IN ('addresses', 'packets', 'bytes')` ), + check('address_maad_stats_bucket_check', sql`${table.bucketEnd} > ${table.bucketStart}`), + check('address_maad_stats_total_addrs_check', sql`${table.totalAddrs} >= 0`), check( + 'address_maad_stats_zero_weight_addrs_check', + sql`${table.zeroWeightAddrs} >= 0 AND (${table.measure} <> 'addresses' OR ${table.zeroWeightAddrs} = 0)` + ), + check( + 'address_maad_stats_prefix_length_check', + sql`${table.maxPrefixLength} >= ${table.minPrefixLength}` + ), + check( + 'address_maad_stats_tau_check', + sql`typeof(${table.tau}) IN ('null', 'blob') AND length(${table.tau}) > 0 AND length(${table.tau}) % 4 = 0` + ), + check( + 'address_maad_stats_tau_sd_check', + sql`typeof(${table.tauSd}) = typeof(${table.tau}) AND length(${table.tauSd}) IS length(${table.tau})` + ), + check( + 'address_maad_stats_spectrum_check', + sql`typeof(${table.spectrum}) IN ('null', 'blob') AND length(${table.spectrum}) % 8 = 0` + ), + check( + 'address_maad_stats_measure_spectrum_check', + sql`${table.measure} = 'addresses' OR ${table.spectrum} IS NULL` + ), - 'address_structure_stats_measure_spectrum_check', - sql`${table.measure} = 'addresses' OR ${table.structureKind} <> 'spectrum'` - ) + check('address_maad_stats_tau_d0_check', sql`(${table.tau} IS NULL) = (${table.d0} IS NULL)`) ] ); + +export const maadQGrid = sqliteTable( + 'maad_q_grid', + { + ipVersion: integer('ip_version').primaryKey(), + qMin: real('q_min').notNull(), + qStep: real('q_step').notNull(), + qCount: integer('q_count').notNull() + }, + (table) => [ + check('maad_q_grid_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), + check('maad_q_grid_q_step_check', sql`${table.qStep} > 0`), + check('maad_q_grid_q_count_check', sql`${table.qCount} > 0`) + ] +); ``` ## apps/web/src/lib/server/netflow-v3.ts Status: modified; +88; −16 ```diff @@ -13,16 +13,18 @@ type MaadIpVersion, type MaadMeasure } from '$lib/types/types'; +import type { ReadonlyDatasetDb, SourceDefinition } from '$lib/server/datasets'; +import type { SpectrumPoint, StructureFunctionPoint } from '$lib/types/types'; + +export type MaadBlob = Uint8Array | ArrayBuffer | number[] | null; -import type { SourceDefinition } from '$lib/server/datasets'; -import type { StructureFunctionPoint } from '$lib/types/types'; -type RawStructureFunctionPoint = { - q: number; - tau?: number; - tauTilde?: number; - sd?: number; - s?: number; -}; +export interface MaadQGridRow { + ipVersion: MaadIpVersion; + qMin: number; + qStep: number; + qCount: number; +} + export interface AggregateStatsParams { routers: string[]; granularity: IpGranularity; @@ -360,12 +362,82 @@ return FIVE_MINUTE_GRANULARITY; } -export function normalizeStructurePoints( +export class MaadDataError extends Error { - points: RawStructureFunctionPoint[] + name = 'MaadDataError'; +} + +export function decodeF32(value: MaadBlob): Float32Array | null { + if (value === null) return null; + if (Array.isArray(value)) return decodeF32(Uint8Array.from(value)); + if (value instanceof ArrayBuffer) return decodeF32(new Uint8Array(value)); + + if (value.byteLength % 4 !== 0) { + throw new MaadDataError(`MAAD blob length ${value.byteLength} is not a multiple of 4 bytes`); + } + const count = value.byteLength / 4; + const view = new DataView(value.buffer, value.byteOffset, value.byteLength); + const out = new Float32Array(count); + for (let i = 0; i < count; i++) { + out[i] = view.getFloat32(i * 4, true); + } + return out; +} + +export async function getMaadQGrid( + db: ReadonlyDatasetDb, + ipVersion: MaadIpVersion +): Promise { + const row = await db.get( + `SELECT + ip_version AS ipVersion, + q_min AS qMin, + q_step AS qStep, + q_count AS qCount + FROM maad_q_grid + WHERE ip_version = ?`, + [ipVersion] + ); + return row ?? null; +} + +export function buildStructurePoints( + tau: MaadBlob, + tauSd: MaadBlob, + qGrid: MaadQGridRow | null ): StructureFunctionPoint[] { - return points.map((point) => ({ + const tauValues = decodeF32(tau); + if (tauValues === null) return []; + if (qGrid === null) { + throw new MaadDataError('MAAD structure function is stored without a q grid'); + } + + const sdValues = decodeF32(tauSd); - q: point.q, - tau: point.tau ?? point.tauTilde ?? 0, - sd: point.sd ?? point.s ?? 0 + if (tauValues.length !== qGrid.qCount || sdValues?.length !== qGrid.qCount) { + throw new MaadDataError( - })); + `MAAD tau and tau_sd must each hold ${qGrid.qCount} values for IPv${qGrid.ipVersion}` + ); + } + const points: StructureFunctionPoint[] = []; + for (let i = 0; i < tauValues.length; i++) { + points.push({ + q: qGrid.qMin + i * qGrid.qStep, + tau: tauValues[i], + sd: sdValues[i] + }); + } + return points; } + +export function buildSpectrumPoints(spectrum: MaadBlob): SpectrumPoint[] | null { + const values = decodeF32(spectrum); + if (values === null) return null; + if (values.length % 2 !== 0) { + throw new MaadDataError('MAAD spectrum must hold alpha/f pairs'); + } + + const points: SpectrumPoint[] = []; + for (let i = 0; i < values.length; i += 2) { + points.push({ alpha: values[i], f: values[i + 1] }); + } + return points; +} ``` ## apps/web/src/routes/api/netflow/files/\[slug\]/details/\+server.ts Status: modified; +123; −159 ```diff @@ -12,7 +12,9 @@ } from '$lib/types/types'; import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; import { + buildSpectrumPoints, + buildStructurePoints, + getMaadQGrid, - normalizeStructurePoints, parseFlowDirectionParams, parseMaadParams } from '$lib/server/netflow-v3'; @@ -27,41 +29,21 @@ daIpv4Count: number | null; saIpv6Count: number | null; daIpv6Count: number | null; + saTau: Uint8Array | null; - structureJsonSa: string | null; - structureJsonDa: string | null; - spectrumJsonSa: string | null; - spectrumJsonDa: string | null; + saTauSd: Uint8Array | null; + daTau: Uint8Array | null; + daTauSd: Uint8Array | null; + saSpectrum: Uint8Array | null; + daSpectrum: Uint8Array | null; }; -function parseStructure(raw: string | null): StructureFunctionPoint[] | null { - if (!raw) return null; - - try { - return normalizeStructurePoints(JSON.parse(raw) as StructureFunctionPoint[]); - } catch (error) { - console.error('Failed to parse structure JSON from database:', error); - return null; - } -} - -function parseSpectrum(raw: string | null): SpectrumPoint[] | null { - if (!raw) return null; - - try { - return JSON.parse(raw) as SpectrumPoint[]; - } catch (error) { - console.error('Failed to parse spectrum JSON from database:', error); - return null; - } -} - function buildStructureData( slug: string, router: string, addressType: 'Source' | 'Destination', - points: StructureFunctionPoint[] | null + points: StructureFunctionPoint[] ): StructureFunctionData | null { + if (points.length === 0) return null; - if (!points || points.length === 0) return null; const qValues = points.map((point) => point.q); const qRange = @@ -149,129 +131,117 @@ try { return await withDb(dataset, platform, async (db) => { + const qGrid = await getMaadQGrid(db, maad.ipVersion); + const rows = await db.all( `WITH ns AS ( - SELECT - source_id AS router, - bucket_start, + SELECT + source_id AS router, + bucket_start, - MAX(bucket_end) AS bucket_end, - SUM(flows) AS flows, - SUM(flows_tcp) AS flows_tcp, - SUM(flows_udp) AS flows_udp, - SUM(flows_icmp) AS flows_icmp, - SUM(flows_other) AS flows_other, - SUM(packets) AS packets, - SUM(packets_tcp) AS packets_tcp, - SUM(packets_udp) AS packets_udp, - SUM(packets_icmp) AS packets_icmp, - SUM(packets_other) AS packets_other, - SUM(bytes) AS bytes, - SUM(bytes_tcp) AS bytes_tcp, - SUM(bytes_udp) AS bytes_udp, - SUM(bytes_icmp) AS bytes_icmp, - SUM(bytes_other) AS bytes_other, - NULL AS first_timestamp, - NULL AS last_timestamp, - NULL AS msec_first, - NULL AS msec_last, - NULL AS sequence_failures, - MAX(processed_at) AS processed_at - FROM traffic_stats - WHERE granularity = ? - AND bucket_start = ? - AND src_locality = ? - AND dst_locality = ? - GROUP BY source_id, bucket_start - ), - ip AS ( - SELECT - source_id, - bucket_start, - SUM(CASE WHEN address_side = 'source' AND ip_version = 4 THEN unique_address_count ELSE 0 END) AS saIpv4Count, - SUM(CASE WHEN address_side = 'destination' AND ip_version = 4 THEN unique_address_count ELSE 0 END) AS daIpv4Count, - SUM(CASE WHEN address_side = 'source' AND ip_version = 6 THEN unique_address_count ELSE 0 END) AS saIpv6Count, - SUM(CASE WHEN address_side = 'destination' AND ip_version = 6 THEN unique_address_count ELSE 0 END) AS daIpv6Count - FROM address_count_stats - WHERE granularity = ? - AND bucket_start = ? - AND src_locality = ? - AND dst_locality = ? - GROUP BY source_id, bucket_start - ), - st AS ( - SELECT - source_id, - bucket_start, - MAX(CASE WHEN address_side = 'source' THEN values_json END) AS structureJsonSa, - MAX(CASE WHEN address_side = 'destination' THEN values_json END) AS structureJsonDa - FROM address_structure_stats - WHERE granularity = ? - AND bucket_start = ? - AND ip_version = ? - AND src_locality = ? - AND dst_locality = ? - AND measure = ? - AND structure_kind = 'structure' - GROUP BY source_id, bucket_start - ), - sp AS ( - SELECT - source_id, - bucket_start, - MAX(CASE WHEN address_side = 'source' THEN values_json END) AS spectrumJsonSa, - MAX(CASE WHEN address_side = 'destination' THEN values_json END) AS spectrumJsonDa - FROM address_structure_stats - WHERE granularity = ? - AND bucket_start = ? - AND ip_version = ? - AND src_locality = ? - AND dst_locality = ? - AND measure = ? - AND structure_kind = 'spectrum' - GROUP BY source_id, bucket_start - ) - SELECT - ns.*, - pi.input_locator AS file_path, - pi.input_kind, - pi.status AS input_status, - pi.error_message AS input_error_message, - COALESCE(ns.processed_at, pi.processed_at, pi.discovered_at) AS processed_at, - ip.saIpv4Count, - ip.daIpv4Count, - ip.saIpv6Count, - ip.daIpv6Count, - st.structureJsonSa, - st.structureJsonDa, - sp.spectrumJsonSa, - sp.spectrumJsonDa - FROM ns - LEFT JOIN ( + MAX(bucket_end) AS bucket_end, + SUM(flows) AS flows, + SUM(flows_tcp) AS flows_tcp, + SUM(flows_udp) AS flows_udp, + SUM(flows_icmp) AS flows_icmp, + SUM(flows_other) AS flows_other, + SUM(packets) AS packets, + SUM(packets_tcp) AS packets_tcp, + SUM(packets_udp) AS packets_udp, + SUM(packets_icmp) AS packets_icmp, + SUM(packets_other) AS packets_other, + SUM(bytes) AS bytes, + SUM(bytes_tcp) AS bytes_tcp, + SUM(bytes_udp) AS bytes_udp, + SUM(bytes_icmp) AS bytes_icmp, + SUM(bytes_other) AS bytes_other, + NULL AS first_timestamp, + NULL AS last_timestamp, + NULL AS msec_first, + NULL AS msec_last, + NULL AS sequence_failures, + MAX(processed_at) AS processed_at + FROM traffic_stats + WHERE granularity = ? + AND bucket_start = ? + AND src_locality = ? + AND dst_locality = ? + GROUP BY source_id, bucket_start + ), + ip AS ( + SELECT + source_id, + bucket_start, + SUM(CASE WHEN address_side = 'source' AND ip_version = 4 THEN unique_address_count ELSE 0 END) AS saIpv4Count, + SUM(CASE WHEN address_side = 'destination' AND ip_version = 4 THEN unique_address_count ELSE 0 END) AS daIpv4Count, + SUM(CASE WHEN address_side = 'source' AND ip_version = 6 THEN unique_address_count ELSE 0 END) AS saIpv6Count, + SUM(CASE WHEN address_side = 'destination' AND ip_version = 6 THEN unique_address_count ELSE 0 END) AS daIpv6Count + FROM address_count_stats + WHERE granularity = ? + AND bucket_start = ? + AND src_locality = ? + AND dst_locality = ? + GROUP BY source_id, bucket_start + ), + maad AS ( + SELECT + source_id, + bucket_start, + MAX(CASE WHEN address_side = 'source' THEN tau END) AS saTau, + MAX(CASE WHEN address_side = 'source' THEN tau_sd END) AS saTauSd, + MAX(CASE WHEN address_side = 'destination' THEN tau END) AS daTau, + MAX(CASE WHEN address_side = 'destination' THEN tau_sd END) AS daTauSd, + MAX(CASE WHEN address_side = 'source' THEN spectrum END) AS saSpectrum, + MAX(CASE WHEN address_side = 'destination' THEN spectrum END) AS daSpectrum + FROM address_maad_stats + WHERE granularity = ? + AND bucket_start = ? + AND ip_version = ? + AND src_locality = ? + AND dst_locality = ? + AND measure = ? + GROUP BY source_id, bucket_start + ) SELECT + ns.*, + pi.input_locator AS file_path, + pi.input_kind, + pi.status AS input_status, + pi.error_message AS input_error_message, + COALESCE(ns.processed_at, pi.processed_at, pi.discovered_at) AS processed_at, + ip.saIpv4Count, + ip.daIpv4Count, + ip.saIpv6Count, + ip.daIpv6Count, - source_id, - bucket_start, - MIN(input_locator) AS input_locator, - MIN(input_kind) AS input_kind, - MAX(status) AS status, - MAX(error_message) AS error_message, - MAX(discovered_at) AS discovered_at, + maad.saTau, + maad.saTauSd, + maad.daTau, + maad.daTauSd, + maad.saSpectrum, + maad.daSpectrum + FROM ns + LEFT JOIN ( + SELECT - MAX(processed_at) AS processed_at - FROM processed_inputs - WHERE bucket_start = ? - GROUP BY source_id, bucket_start - ) pi - ON pi.source_id = ns.router - AND pi.bucket_start = ns.bucket_start - LEFT JOIN ip - ON ip.source_id = ns.router - AND ip.bucket_start = ns.bucket_start - LEFT JOIN st - ON st.source_id = ns.router - AND st.bucket_start = ns.bucket_start - LEFT JOIN sp - ON sp.source_id = ns.router - AND sp.bucket_start = ns.bucket_start - ORDER BY ns.router`, + source_id, + bucket_start, + MIN(input_locator) AS input_locator, + MIN(input_kind) AS input_kind, + MAX(status) AS status, + MAX(error_message) AS error_message, + MAX(discovered_at) AS discovered_at, + MAX(processed_at) AS processed_at + FROM processed_inputs + WHERE bucket_start = ? + GROUP BY source_id, bucket_start + ) pi + ON pi.source_id = ns.router + AND pi.bucket_start = ns.bucket_start + LEFT JOIN ip + ON ip.source_id = ns.router + AND ip.bucket_start = ns.bucket_start + LEFT JOIN maad + ON maad.source_id = ns.router + AND maad.bucket_start = ns.bucket_start + ORDER BY ns.router`, [ FIVE_MINUTES, bucketStart, @@ -287,12 +257,6 @@ flowDirection.srcLocality, flowDirection.dstLocality, maad.measure, - FIVE_MINUTES, - bucketStart, - maad.ipVersion, - flowDirection.srcLocality, - flowDirection.dstLocality, - maad.measure, bucketStart ] ); @@ -302,10 +266,10 @@ } const routers: NetflowFileDetailsRouter[] = rows.map((row) => { - const structureSourcePoints = parseStructure(row.structureJsonSa); - const structureDestinationPoints = parseStructure(row.structureJsonDa); - const spectrumSourcePoints = parseSpectrum(row.spectrumJsonSa); - const spectrumDestinationPoints = parseSpectrum(row.spectrumJsonDa); + const structureSourcePoints = buildStructurePoints(row.saTau, row.saTauSd, qGrid); + const structureDestinationPoints = buildStructurePoints(row.daTau, row.daTauSd, qGrid); + const spectrumSourcePoints = buildSpectrumPoints(row.saSpectrum); + const spectrumDestinationPoints = buildSpectrumPoints(row.daSpectrum); return { summary: { ``` ## apps/web/src/routes/api/netflow/files/\[slug\]/spectrum/\+server.ts Status: modified; +15; −31 ```diff @@ -1,7 +1,8 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import type { SpectrumData, SpectrumPoint } from '$lib/types/types'; +import type { SpectrumData } from '$lib/types/types'; import { + buildSpectrumPoints, parseFlowDirectionParams, parseMaadParams, spectrumMeasureError @@ -11,7 +12,7 @@ const FIVE_MINUTES = '5m'; type SpectrumRow = { - valuesJson: string | null; + spectrum: Uint8Array | null; }; export const GET: RequestHandler = async ({ params, url, platform }) => { @@ -61,18 +62,17 @@ return await withDb(dataset, platform, async (db) => { const row = await db.get( `SELECT - values_json AS valuesJson + spectrum AS spectrum + FROM address_maad_stats + WHERE source_id = ? + AND granularity = ? + AND bucket_start = ? + AND ip_version = ? - FROM address_structure_stats - WHERE source_id = ? - AND granularity = ? - AND bucket_start = ? - AND ip_version = ? - AND src_locality = ? - AND dst_locality = ? - AND address_side = ? - AND measure = ? - AND structure_kind = 'spectrum' - LIMIT 1`, + AND src_locality = ? + AND dst_locality = ? + AND address_side = ? + AND measure = ? + LIMIT 1`, [ router, FIVE_MINUTES, @@ -92,23 +92,7 @@ ); } - const rawSpectrum = row.valuesJson; - if (!rawSpectrum) { - return json( - { error: `Spectrum statistics not found for router ${router} at ${slug}` }, - { status: 404 } + const data = buildSpectrumPoints(row.spectrum) ?? []; - ); - } - - let data: SpectrumPoint[] = []; - - try { - data = JSON.parse(rawSpectrum) as SpectrumPoint[]; - } catch (error) { - console.error('Failed to parse spectrum JSON from database:', error); - return json({ error: 'Failed to parse spectrum statistics' }, { status: 500 }); - } - if (data.length === 0) { return json( { error: `Spectrum statistics not found for router ${router} at ${slug}` }, ``` ## apps/web/src/routes/api/netflow/files/\[slug\]/structure/\+server.ts Status: modified; +19; −31 ```diff @@ -1,9 +1,10 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import type { StructureFunctionData, StructureFunctionPoint } from '$lib/types/types'; +import type { StructureFunctionData } from '$lib/types/types'; import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; import { + buildStructurePoints, + getMaadQGrid, - normalizeStructurePoints, parseFlowDirectionParams, parseMaadParams } from '$lib/server/netflow-v3'; @@ -11,7 +12,8 @@ const FIVE_MINUTES = '5m'; type StructureRow = { + tau: Uint8Array | null; + tauSd: Uint8Array | null; - valuesJson: string | null; }; export const GET: RequestHandler = async ({ params, url, platform }) => { @@ -56,18 +58,18 @@ return await withDb(dataset, platform, async (db) => { const row = await db.get( `SELECT + tau AS tau, + tau_sd AS tauSd + FROM address_maad_stats + WHERE source_id = ? + AND granularity = ? - values_json AS valuesJson - FROM address_structure_stats - WHERE source_id = ? - AND granularity = ? - AND bucket_start = ? - AND ip_version = ? - AND src_locality = ? - AND dst_locality = ? - AND address_side = ? - AND measure = ? - AND structure_kind = 'structure' - LIMIT 1`, + AND bucket_start = ? + AND ip_version = ? + AND src_locality = ? + AND dst_locality = ? + AND address_side = ? + AND measure = ? + LIMIT 1`, [ router, FIVE_MINUTES, @@ -87,23 +89,9 @@ ); } - const rawStructure = row.valuesJson; - if (!rawStructure) { - return json( - { error: `Structure statistics not found for router ${router} at ${slug}` }, + const qGrid = await getMaadQGrid(db, maad.ipVersion); - { status: 404 } - ); - } - let data: StructureFunctionPoint[] = []; - - try { - data = normalizeStructurePoints(JSON.parse(rawStructure) as StructureFunctionPoint[]); - } catch (error) { - console.error('Failed to parse structure JSON from database:', error); - return json({ error: 'Failed to parse structure statistics' }, { status: 500 }); - } + const data = buildStructurePoints(row.tau, row.tauSd, qGrid); - if (data.length === 0) { return json( { error: `Structure statistics not found for router ${router} at ${slug}` }, ``` ## apps/web/src/routes/api/netflow/spectrum-stats/\+server.ts Status: modified; +33; −66 ```diff @@ -1,10 +1,14 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import type { SpectrumPoint } from '$lib/types/types'; import type { SpectrumStatsPayload, SpectrumStatsResponse } from '$lib/types/spectrum-stats'; import { buildCoverageTimelines } from '$lib/server/db/coverage'; import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; +import { + buildSpectrumPoints, + parseMaadStatsParams, + placeholders, + spectrumMeasureError +} from '$lib/server/netflow-v3'; -import { parseMaadStatsParams, placeholders, spectrumMeasureError } from '$lib/server/netflow-v3'; type SpectrumStatsRow = SpectrumStatsPayload & { router: string; @@ -16,38 +20,10 @@ router: string; bucketStart: number; bucketEnd: number; - spectrumSaJson: string | null; - spectrumDaJson: string | null; + saSpectrum: Uint8Array | null; + daSpectrum: Uint8Array | null; }; -function isSpectrumPoint(value: unknown): value is SpectrumPoint { - return ( - typeof value === 'object' && - value !== null && - 'alpha' in value && - 'f' in value && - typeof value.alpha === 'number' && - typeof value.f === 'number' - ); -} - -function parseSpectrumPoints( - valuesJson: string | null, - router: string, - bucketStart: number -): SpectrumPoint[] | null { - if (valuesJson === null) return null; - - try { - const values: unknown = JSON.parse(valuesJson); - if (!Array.isArray(values)) return null; - return values.filter(isSpectrumPoint); - } catch (error) { - console.error('Failed to parse spectrum values_json:', { router, bucketStart, error }); - return null; - } -} - export const GET: RequestHandler = async ({ url, platform }) => { const params = parseMaadStatsParams(url); if ('error' in params) { @@ -62,7 +38,7 @@ try { const dataset = await getRequestedDataset(url, platform); return await withDatasetDb(dataset, platform, async ({ db }) => { - const tableName = 'address_structure_stats'; + const tableName = 'address_maad_stats'; const sourceColumn = 'source_id'; const queryParams = [ granularity, @@ -76,41 +52,32 @@ ]; const query = ` + SELECT + ${sourceColumn} AS router, + bucket_start AS bucketStart, + MAX(bucket_end) AS bucketEnd, + MAX(CASE WHEN address_side = 'source' THEN spectrum END) AS saSpectrum, + MAX(CASE WHEN address_side = 'destination' THEN spectrum END) AS daSpectrum + FROM ${tableName} + WHERE granularity = ? + AND ${sourceColumn} IN (${placeholders(routers)}) + AND src_locality = ? + AND dst_locality = ? + AND bucket_start >= ? + AND bucket_start < ? + AND ip_version = ? + AND measure = ? - SELECT - ${sourceColumn} AS router, - bucket_start AS bucketStart, + GROUP BY ${sourceColumn}, bucket_start + `; - MAX(bucket_end) AS bucketEnd, - MAX(CASE WHEN address_side = 'source' THEN values_json END) AS spectrumSaJson, - MAX(CASE WHEN address_side = 'destination' THEN values_json END) AS spectrumDaJson - FROM ${tableName} - WHERE granularity = ? - AND ${sourceColumn} IN (${placeholders(routers)}) - AND src_locality = ? - AND dst_locality = ? - AND bucket_start >= ? - AND bucket_start < ? - AND ip_version = ? - AND measure = ? - AND structure_kind = 'spectrum' - GROUP BY ${sourceColumn}, bucket_start - `; const rawRows = await db.all(query, queryParams); - const rows: SpectrumStatsRow[] = rawRows.flatMap((row) => { - const spectrumSa = parseSpectrumPoints(row.spectrumSaJson, row.router, row.bucketStart); - const spectrumDa = parseSpectrumPoints(row.spectrumDaJson, row.router, row.bucketStart); - return spectrumSa === null && spectrumDa === null - ? [] - : [ - { - router: row.router, - bucketStart: row.bucketStart, - bucketEnd: row.bucketEnd, - spectrumSa: spectrumSa ?? [], - spectrumDa: spectrumDa ?? [] - } - ]; + const rows: SpectrumStatsRow[] = rawRows.map((row) => ({ - }); + router: row.router, + bucketStart: row.bucketStart, + bucketEnd: row.bucketEnd, + spectrumSa: buildSpectrumPoints(row.saSpectrum) ?? [], + spectrumDa: buildSpectrumPoints(row.daSpectrum) ?? [] + })); const timelines = await buildCoverageTimelines({ db, granularity, ``` ## apps/web/src/routes/api/netflow/structure-stats/\+server.ts Status: modified; +35; −76 ```diff @@ -1,11 +1,11 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; -import type { StructureFunctionPoint } from '$lib/types/types'; import type { StructureStatsPayload, StructureStatsResponse } from '$lib/types/structure-stats'; import { buildCoverageTimelines } from '$lib/server/db/coverage'; import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; import { - normalizeStructurePoints, + buildStructurePoints, + getMaadQGrid, parseMaadStatsParams, placeholders } from '$lib/server/netflow-v3'; @@ -20,48 +20,12 @@ router: string; bucketStart: number; bucketEnd: number; - structureSaJson: string | null; - structureDaJson: string | null; -}; - -type RawStructurePoint = { - q: number; - tau?: number; - tauTilde?: number; - sd?: number; - s?: number; + saTau: Uint8Array | null; + saTauSd: Uint8Array | null; + daTau: Uint8Array | null; + daTauSd: Uint8Array | null; }; -function isRawStructurePoint(value: unknown): value is RawStructurePoint { - return ( - typeof value === 'object' && - value !== null && - 'q' in value && - typeof value.q === 'number' && - (!('tau' in value) || typeof value.tau === 'number') && - (!('tauTilde' in value) || typeof value.tauTilde === 'number') && - (!('sd' in value) || typeof value.sd === 'number') && - (!('s' in value) || typeof value.s === 'number') - ); -} - -function parseStructurePoints( - valuesJson: string | null, - router: string, - bucketStart: number -): StructureFunctionPoint[] | null { - if (valuesJson === null) return null; - - try { - const values: unknown = JSON.parse(valuesJson); - if (!Array.isArray(values)) return null; - return normalizeStructurePoints(values.filter(isRawStructurePoint)); - } catch (error) { - console.error('Failed to parse structure values_json:', { router, bucketStart, error }); - return null; - } -} - export const GET: RequestHandler = async ({ url, platform }) => { const params = parseMaadStatsParams(url); if ('error' in params) { @@ -72,7 +36,9 @@ try { const dataset = await getRequestedDataset(url, platform); return await withDatasetDb(dataset, platform, async ({ db }) => { - const tableName = 'address_structure_stats'; + const qGrid = await getMaadQGrid(db, ipVersion); + + const tableName = 'address_maad_stats'; const sourceColumn = 'source_id'; const queryParams = [ granularity, @@ -86,41 +52,34 @@ ]; const query = ` + SELECT + ${sourceColumn} AS router, + bucket_start AS bucketStart, + MAX(bucket_end) AS bucketEnd, - SELECT - ${sourceColumn} AS router, - bucket_start AS bucketStart, - MAX(bucket_end) AS bucketEnd, - MAX(CASE WHEN address_side = 'source' THEN values_json END) AS structureSaJson, - MAX(CASE WHEN address_side = 'destination' THEN values_json END) AS structureDaJson - FROM ${tableName} - WHERE granularity = ? - AND ${sourceColumn} IN (${placeholders(routers)}) - AND src_locality = ? + MAX(CASE WHEN address_side = 'source' THEN tau END) AS saTau, - AND dst_locality = ? - AND bucket_start >= ? - AND bucket_start < ? + MAX(CASE WHEN address_side = 'source' THEN tau_sd END) AS saTauSd, + MAX(CASE WHEN address_side = 'destination' THEN tau END) AS daTau, + MAX(CASE WHEN address_side = 'destination' THEN tau_sd END) AS daTauSd + FROM ${tableName} + WHERE granularity = ? + AND ${sourceColumn} IN (${placeholders(routers)}) + AND src_locality = ? + AND dst_locality = ? + AND bucket_start >= ? + AND bucket_start < ? + AND ip_version = ? + AND measure = ? + GROUP BY ${sourceColumn}, bucket_start + `; - AND ip_version = ? - AND measure = ? - AND structure_kind = 'structure' - GROUP BY ${sourceColumn}, bucket_start - `; const rawRows = await db.all(query, queryParams); - const rows: StructureStatsRow[] = rawRows.flatMap((row) => { - const structureSa = parseStructurePoints(row.structureSaJson, row.router, row.bucketStart); + const rows: StructureStatsRow[] = rawRows.map((row) => ({ + router: row.router, + bucketStart: row.bucketStart, + bucketEnd: row.bucketEnd, + structureSa: buildStructurePoints(row.saTau, row.saTauSd, qGrid), + structureDa: buildStructurePoints(row.daTau, row.daTauSd, qGrid) + })); - const structureDa = parseStructurePoints(row.structureDaJson, row.router, row.bucketStart); - return structureSa === null && structureDa === null - ? [] - : [ - { - router: row.router, - bucketStart: row.bucketStart, - bucketEnd: row.bucketEnd, - structureSa: structureSa ?? [], - structureDa: structureDa ?? [] - } - ]; - }); const timelines = await buildCoverageTimelines({ db, granularity, ``` ## apps/web/tests/e2e/playwright-fixture.js Status: modified; +3; −0 ```diff @@ -31,6 +31,9 @@ source_id, granularity, bucket_start, bucket_end, coverage_state, observed_units, expected_units, rejected_units ) VALUES ('fixture-router', '5m', 1740823200, 1740823500, 'complete', 1, 1, 0); + INSERT INTO maad_q_grid (ip_version, q_min, q_step, q_count) VALUES + (4, -0.5, 0.125, 33), + (6, -0.5, 0.125, 33); `); database.close(); return { databasePath, fixtureDirectory }; ``` ## apps/web/tests/lib/server/migrations.test.ts Status: modified; +103; −16 ```diff @@ -3,6 +3,7 @@ import { fileURLToPath } from 'node:url'; import Database from 'better-sqlite3'; import { describe, expect, it } from 'vitest'; +import { localSchemaSql } from '../../../src/lib/server/db/local-schema'; const migrationsDirectory = fileURLToPath(new URL('../../../drizzle', import.meta.url)); @@ -20,6 +21,12 @@ .sort(); } +function migrationIndexContaining(migrations: string[], needle: string): number { + return migrations.findIndex((fileName) => + fs.readFileSync(path.join(migrationsDirectory, fileName), 'utf8').includes(needle) + ); +} + function applyMigration(database: Database.Database, fileName: string): void { database.exec(fs.readFileSync(path.join(migrationsDirectory, fileName), 'utf8')); } @@ -155,7 +162,7 @@ ['source_id', 'granularity', 'src_locality', 'dst_locality', 'bucket_start'] ], [ + 'idx_address_maad_stats_key', - 'idx_address_structure_stats_timeseries', [ 'source_id', 'granularity', @@ -163,8 +170,8 @@ 'dst_locality', 'ip_version', 'measure', - 'structure_kind', + 'bucket_start', + 'address_side' - 'bucket_start' ] ] ]); @@ -184,17 +191,100 @@ for (const indexName of [ 'idx_traffic_stats_query', 'idx_address_count_stats_query', + 'idx_address_maad_stats_bucket' - 'idx_address_structure_stats_query' ]) { expect(indexColumns(database, indexName), indexName).not.toHaveLength(0); } expect(indexColumns(database, 'idx_protocol_stats_query')).toHaveLength(0); expect(indexColumns(database, 'idx_port_count_stats_query')).toHaveLength(0); + + const maadColumns = database + .prepare('PRAGMA table_info(address_maad_stats)') + .all() + .map((column) => (column as { name: string }).name); + expect(maadColumns).toEqual( + expect.arrayContaining([ + 'total_addrs', + 'zero_weight_addrs', + 'min_prefix_length', + 'max_prefix_length', + 'd0', + 'd1', + 'd2', + 'tau', + 'tau_sd', + 'spectrum' + ]) + ); + expect(maadColumns).not.toEqual( + expect.arrayContaining(['values_json', 'metadata_json', 'structure_kind', 'processed_at']) + ); + + const qGridColumns = database + .prepare('PRAGMA table_info(maad_q_grid)') + .all() + .map((column) => (column as { name: string }).name); + expect(qGridColumns).toEqual(['ip_version', 'q_min', 'q_step', 'q_count']); + + expect( + database + .prepare("SELECT name FROM sqlite_master WHERE name = 'address_structure_stats'") + .all() + ).toHaveLength(0); } finally { database.close(); } }); + it.each([ + [ + 'migrated', + (database: Database.Database) => + migrationFiles().forEach((file) => applyMigration(database, file)) + ], + ['local', (database: Database.Database) => database.exec(localSchemaSql)] + ])('rejects malformed MAAD blobs in the %s schema', (_schema, createSchema) => { + const database = new Database(':memory:'); + + try { + createSchema(database); + const insert = database.prepare(` + INSERT INTO address_maad_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, total_addrs, + d0, d1, d2, tau, tau_sd, spectrum + ) VALUES ('edge', '5m', ?, 300, 4, 'all', 'all', 'source', 'addresses', 1, ?, 1, 1, ?, ?, ?) + `); + const bytes = (length: number) => Buffer.alloc(length); + let bucketStart = 0; + const insertRow = ( + tau: Buffer | string | null, + tauSd: Buffer | string | null, + spectrum: Buffer | string | null + ) => insert.run(bucketStart++, tau === null ? null : 1, tau, tauSd, spectrum); + + expect(() => insertRow(bytes(8), bytes(8), bytes(16))).not.toThrow(); + expect(() => insertRow(null, null, bytes(0))).not.toThrow(); + expect(() => insertRow(null, null, null)).not.toThrow(); + + for (const [tau, tauSd, spectrum] of [ + [bytes(0), bytes(0), null], + [bytes(1), bytes(1), null], + [bytes(6), bytes(6), null], + [bytes(8), bytes(4), null], + [bytes(8), null, null], + ['abcd', 'abcd', null], + [bytes(8), 'abcdefgh', null], + [null, null, bytes(4)], + [null, null, 'abcdefgh'] + ] as const) { + expect(() => insertRow(tau, tauSd, spectrum)).toThrow(/CHECK constraint failed/); + } + } finally { + database.close(); + } + }); + it('upgrades the deployed coverage schema without retaining unused bucket-first indexes', () => { const database = new Database(':memory:'); const migrations = migrationFiles(); @@ -264,9 +354,7 @@ it('keeps only locality-independent rollup rows when renaming visibility to locality', () => { const database = new Database(':memory:'); const migrations = migrationFiles(); - const localityMigration = migrations.findIndex((fileName) => - fs.readFileSync(path.join(migrationsDirectory, fileName), 'utf8').includes('src_locality') - ); + const localityMigration = migrationIndexContaining(migrations, 'src_locality'); try { for (const migration of migrations.slice(0, localityMigration)) { applyMigration(database, migration); @@ -305,9 +393,8 @@ it('keeps only locality-independent rows in every stats table', () => { const database = new Database(':memory:'); const migrations = migrationFiles(); + const localityMigration = migrationIndexContaining(migrations, 'src_locality'); + const maadMigration = migrationIndexContaining(migrations, 'address_maad_stats'); - const localityMigration = migrations.findIndex((fileName) => - fs.readFileSync(path.join(migrationsDirectory, fileName), 'utf8').includes('src_locality') - ); const tables = [ 'traffic_stats', 'protocol_stats', @@ -329,7 +416,7 @@ .run(); } - for (const migration of migrations.slice(localityMigration)) { + for (const migration of migrations.slice(localityMigration, maadMigration)) { applyMigration(database, migration); } @@ -347,11 +434,11 @@ it('keys MAAD rows by measure and keeps existing rows as the addresses measure', () => { const database = new Database(':memory:'); const migrations = migrationFiles(); - const measureMigration = migrations.findIndex((fileName) => - fs - .readFileSync(path.join(migrationsDirectory, fileName), 'utf8') - .includes('address_structure_stats_measure_check') + const measureMigration = migrationIndexContaining( + migrations, + 'address_structure_stats_measure_check' ); + const maadMigration = migrationIndexContaining(migrations, 'address_maad_stats'); try { for (const migration of migrations.slice(0, measureMigration)) { applyMigration(database, migration); @@ -366,7 +453,7 @@ ) .run(); + for (const migration of migrations.slice(measureMigration, maadMigration)) { - for (const migration of migrations.slice(measureMigration)) { applyMigration(database, migration); } ``` ## apps/web/tests/lib/server/netflow-v3.test.ts Status: modified; +64; −4 ```diff @@ -1,8 +1,11 @@ import { describe, expect, it } from 'vitest'; import { + buildSpectrumPoints, + buildStructurePoints, + decodeF32, + MaadDataError, getNetflowSchemaVersion, groupByToGranularity, - normalizeStructurePoints, parseAggregateStatsParams, parseIpGranularity, parseIpGranularityOrDefault, @@ -16,6 +19,10 @@ resolveSourceIds } from '../../../src/lib/server/netflow-v3'; +function f32(values: number[]): Uint8Array { + return new Uint8Array(Float32Array.from(values).buffer); +} + describe('netflow v3 helpers', () => { it('is v3-only', () => { expect(getNetflowSchemaVersion()).toBe('v3'); @@ -215,12 +222,65 @@ ).toEqual({ error: 'No routers selected', status: 400 }); }); - it('normalizes structure points from MAAD variants', () => { - expect(normalizeStructurePoints([{ q: 1, tauTilde: 2, s: 3 }])).toEqual([ - { q: 1, tau: 2, sd: 3 } + it('decodes little-endian f32 blobs from every accepted input shape', () => { + expect(decodeF32(null)).toBeNull(); + expect(decodeF32(f32([]))).toEqual(new Float32Array([])); + expect(decodeF32(f32([1, 2, 3]))).toEqual(new Float32Array([1, 2, 3])); + expect(decodeF32(new Float32Array([1.5, -2.25]).buffer as ArrayBuffer)).toEqual( + new Float32Array([1.5, -2.25]) + ); + expect(decodeF32([0, 0, 128, 63])).toEqual(new Float32Array([1])); + + const padded = Buffer.concat([Buffer.from([0, 0, 0]), Buffer.from(f32([1, 2]))]); + const misaligned = padded.subarray(3); + expect(misaligned.byteOffset % 4).not.toBe(0); + expect(decodeF32(misaligned)).toEqual(new Float32Array([1, 2])); + }); + + it('rejects f32 blobs whose length is not a multiple of four bytes', () => { + expect(() => decodeF32(Uint8Array.from([0, 0, 128]))).toThrow(MaadDataError); + expect(() => decodeF32(Uint8Array.from([0, 0, 128, 63, 0]))).toThrow(MaadDataError); + }); + + it('builds structure points from tau/tau_sd blobs and a q grid, or an empty array for a null result', () => { + const qGrid = { ipVersion: 4 as const, qMin: -0.5, qStep: 0.5, qCount: 2 }; + expect(buildStructurePoints(f32([0.2, 0.4]), f32([0.01, 0.02]), qGrid)).toEqual([ + { q: -0.5, tau: Math.fround(0.2), sd: Math.fround(0.01) }, + { q: 0, tau: Math.fround(0.4), sd: Math.fround(0.02) } ]); + expect(buildStructurePoints(null, null, qGrid)).toEqual([]); + expect(buildStructurePoints(null, null, null)).toEqual([]); }); + it('rejects structure blobs that are missing a q grid or disagree with its length', () => { + const qGrid = { ipVersion: 4 as const, qMin: -0.5, qStep: 0.5, qCount: 33 }; + const tau = f32(Array.from({ length: 33 }, (_, i) => i / 10)); + expect(buildStructurePoints(tau, tau, qGrid)).toHaveLength(33); + expect(() => buildStructurePoints(tau, tau, null)).toThrow(MaadDataError); + expect(() => buildStructurePoints(f32([0.25]), f32([0.5]), qGrid)).toThrow(MaadDataError); + expect(() => buildStructurePoints(tau, null, qGrid)).toThrow(MaadDataError); + expect(() => buildStructurePoints(tau, f32([0.5]), qGrid)).toThrow(MaadDataError); + expect(() => buildStructurePoints(Uint8Array.from([0, 0, 128]), tau, qGrid)).toThrow( + MaadDataError + ); + }); + + it('builds spectrum points from interleaved alpha/f blobs, and distinguishes empty from not computed', () => { + expect(buildSpectrumPoints(f32([1, 2, 3, 4]))).toEqual([ + { alpha: 1, f: 2 }, + { alpha: 3, f: 4 } + ]); + expect(buildSpectrumPoints(f32([]))).toEqual([]); + expect(buildSpectrumPoints(null)).toBeNull(); + }); + + it('rejects spectrum blobs that do not hold whole alpha/f pairs', () => { + expect(() => buildSpectrumPoints(f32([1, 2, 3]))).toThrow(MaadDataError); + expect(() => buildSpectrumPoints(Uint8Array.from([0, 0, 128, 63, 0, 0]))).toThrow( + MaadDataError + ); + }); + it('resolves additive sources to one disjoint physical cover', () => { const definitions = [ { sourceId: 'cc_ir1_gw', members: ['cc_ir1_gw'] }, ``` ## apps/web/tests/performance/seed-large-db.ts Status: modified; +70; −18 ```diff @@ -34,13 +34,17 @@ 'idx_protocol_stats_timeseries', 'idx_address_count_stats_query', 'idx_address_count_stats_timeseries', - 'idx_address_structure_stats_query', + 'idx_address_maad_stats_key', - 'idx_address_structure_stats_timeseries', + 'idx_address_maad_stats_bucket', 'idx_port_count_stats_timeseries' ]) { db.exec(`DROP INDEX ${index}`); } +function f32Buffer(values: number[]): Buffer { + return Buffer.from(Float32Array.from(values).buffer); +} + db.exec(` CREATE TEMP TABLE perf_sources (source_id TEXT PRIMARY KEY) WITHOUT ROWID; WITH RECURSIVE sources(value) AS ( @@ -193,25 +197,73 @@ CROSS JOIN (SELECT 'low' AS port_range UNION ALL SELECT 'high') WHERE bucket_start >= hourly_start AND bucket_start < hourly_end; + INSERT INTO maad_q_grid (ip_version, q_min, q_step, q_count) VALUES - INSERT INTO address_structure_stats ( + (4, -0.5, 0.125, 33), + (6, -0.5, 0.125, 33); +`); + +const maadRows = db + .prepare( + ` + SELECT + perf_sources.source_id AS sourceId, + perf_hour_buckets.bucket_start AS bucketStart, + perf_scopes.src_locality AS srcLocality, + perf_scopes.dst_locality AS dstLocality, + address_side.value AS addressSide + FROM perf_sources + CROSS JOIN perf_hour_buckets + CROSS JOIN perf_scopes + CROSS JOIN (SELECT 'source' AS value UNION ALL SELECT 'destination') AS address_side + WHERE perf_scopes.ip_version = 4 + AND perf_hour_buckets.bucket_start >= perf_scopes.hourly_start + AND perf_hour_buckets.bucket_start < perf_scopes.hourly_end + ` + ) + .all() as { + sourceId: string; + bucketStart: number; + srcLocality: string; + dstLocality: string; + addressSide: string; +}[]; + +const maadTau = f32Buffer(Array.from({ length: 33 }, (_, index) => 0.1 + index * 0.01)); +const maadTauSd = f32Buffer(Array.from({ length: 33 }, () => 0.01)); +const maadSpectrum = f32Buffer( + Array.from({ length: 66 }, (_, index) => (index % 2 === 0 ? 1 + index / 66 : 0.5)) +); + +const insertMaad = db.prepare(` + INSERT INTO address_maad_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, + total_addrs, zero_weight_addrs, min_prefix_length, max_prefix_length, + d0, d1, d2, tau, tau_sd, spectrum + ) VALUES ( + ?, '1h', ?, ? + 3600, 4, - src_locality, dst_locality, address_side, measure, structure_kind, - values_json, metadata_json + ?, ?, ?, 'addresses', + 1000, 0, 8, 24, + 1.5, 1.2, 1.1, ?, ?, ? ) - SELECT source_id, '1h', bucket_start, bucket_start + 3600, 4, - src_locality, dst_locality, address_side, 'addresses', structure_kind, - CASE structure_kind - WHEN 'spectrum' THEN '[{"alpha":0.1,"f":0.2},{"alpha":0.2,"f":0.3},{"alpha":0.3,"f":0.4}]' - ELSE '[{"q":1,"tau":0.2,"sd":0.01},{"q":2,"tau":0.4,"sd":0.02}]' - END, - '{"uniqueAddressCount":1000}' - FROM perf_sources CROSS JOIN perf_hour_buckets CROSS JOIN perf_scopes - CROSS JOIN (SELECT 'source' AS address_side UNION ALL SELECT 'destination') - CROSS JOIN (SELECT 'spectrum' AS structure_kind UNION ALL SELECT 'structure') - WHERE ip_version = 4 - AND bucket_start >= hourly_start AND bucket_start < hourly_end; `); +db.transaction((rows: typeof maadRows) => { + for (const row of rows) { + insertMaad.run( + row.sourceId, + row.bucketStart, + row.bucketStart, + row.srcLocality, + row.dstLocality, + row.addressSide, + maadTau, + maadTauSd, + maadSpectrum + ); + } +})(maadRows); + db.exec(localSchemaSql); db.exec('ANALYZE; VACUUM;'); @@ -222,7 +274,7 @@ 'protocol_stats', 'address_count_stats', 'port_count_stats', - 'address_structure_stats' + 'address_maad_stats' ].map((table) => [table, db.prepare(`SELECT COUNT(*) AS count FROM ${table}`).get()]) ); db.close(); ``` ## apps/web/tests/routes/api-aggregate-routes.test.ts Status: modified; +136; −49 ```diff @@ -16,6 +16,12 @@ vi.mocked(withDatasetDb).mockImplementation(async (_datasetId, _platform, run) => run({ db: db as never, listSources: async () => [], listSourceDefinitions: async () => [] }) ); +} + +const DEFAULT_Q_GRID = { ipVersion: 4, qMin: -0.5, qStep: 0.125, qCount: 33 }; + +function f32(values: number[]): Buffer { + return Buffer.from(Float32Array.from(values).buffer); } describe('aggregate API routes', () => { @@ -290,23 +296,23 @@ }); }); - it('parses spectrum and structure json payloads, tolerating bad json', async () => { + it('decodes spectrum and structure MAAD blobs into points', async () => { - const all = vi + const spectrumAll = vi .fn() .mockResolvedValueOnce([ { router: 'r1', bucketStart: 100, bucketEnd: 200, - spectrumSaJson: '[{"alpha":1,"f":2}]', - spectrumDaJson: 'not-json' + saSpectrum: f32([1, 2]), + daSpectrum: null }, { router: 'r1', bucketStart: 200, bucketEnd: 500, - spectrumSaJson: 'not-json', + saSpectrum: f32([]), + daSpectrum: null - spectrumDaJson: null } ]) .mockResolvedValueOnce([ @@ -328,21 +334,66 @@ expectedUnits: 1, rejectedUnits: 0 } + ]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all: spectrumAll }); + + const spectrumResponse = await getSpectrumStats({ + url: new URL( + 'http://localhost/api/netflow/spectrum-stats?routers=r1&startDate=100&endDate=500' + ) + } as never); + + await expect(spectrumResponse.json()).resolves.toEqual({ + timelines: [ + { + router: 'r1', + buckets: [ + { + bucketStart: 100, + bucketEnd: 200, + coverage: { state: 'complete', observedUnits: 1, expectedUnits: 1 }, + data: { + spectrumSa: [{ alpha: Math.fround(1), f: Math.fround(2) }], + spectrumDa: [] + } + }, + { + bucketStart: 200, + bucketEnd: 500, + coverage: { state: 'complete', observedUnits: 1, expectedUnits: 1 }, + data: { + spectrumSa: [], + spectrumDa: [] + } + } + ] + } + ], + requestedRouters: ['r1'] + }); + + const structureGet = vi.fn().mockResolvedValue({ ...DEFAULT_Q_GRID, qCount: 2 }); + const structureAll = vi - ]) + .fn() .mockResolvedValueOnce([ { router: 'r1', bucketStart: 100, bucketEnd: 200, + saTau: f32([0.2, 0.4]), + saTauSd: f32([0.01, 0.02]), + daTau: null, + daTauSd: null - structureSaJson: '[{"q":1,"tauTilde":2,"sd":0.5}]', - structureDaJson: 'not-json' }, { router: 'r1', bucketStart: 200, bucketEnd: 500, + saTau: null, - structureSaJson: 'not-json', - structureDaJson: null + saTauSd: null, + daTau: null, + daTauSd: null } ]) .mockResolvedValueOnce([ @@ -365,23 +416,15 @@ rejectedUnits: 0 } ]); - vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); - mockDatasetSession({ - all - }); + mockDatasetSession({ all: structureAll, get: structureGet }); - const spectrumResponse = await getSpectrumStats({ - url: new URL( - 'http://localhost/api/netflow/spectrum-stats?routers=r1&startDate=100&endDate=500' - ) - } as never); const structureResponse = await getStructureStats({ url: new URL( 'http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=500' ) } as never); + await expect(structureResponse.json()).resolves.toEqual({ - await expect(spectrumResponse.json()).resolves.toEqual({ timelines: [ { router: 'r1', @@ -391,40 +434,21 @@ bucketEnd: 200, coverage: { state: 'complete', observedUnits: 1, expectedUnits: 1 }, data: { + structureSa: [ + { q: -0.5, tau: Math.fround(0.2), sd: Math.fround(0.01) }, + { q: -0.375, tau: Math.fround(0.4), sd: Math.fround(0.02) } + ], + structureDa: [] - spectrumSa: [{ alpha: 1, f: 2 }], - spectrumDa: [] } }, { bucketStart: 200, bucketEnd: 500, coverage: { state: 'complete', observedUnits: 1, expectedUnits: 1 }, - data: null - } - ] - } - ], - requestedRouters: ['r1'] - }); - await expect(structureResponse.json()).resolves.toEqual({ - timelines: [ - { - router: 'r1', - buckets: [ - { - bucketStart: 100, - bucketEnd: 200, - coverage: { state: 'complete', observedUnits: 1, expectedUnits: 1 }, data: { - structureSa: [{ q: 1, tau: 2, sd: 0.5 }], + structureSa: [], structureDa: [] } - }, - { - bucketStart: 200, - bucketEnd: 500, - coverage: { state: 'complete', observedUnits: 1, expectedUnits: 1 }, - data: null } ] } @@ -433,6 +457,67 @@ }); }); + it('fails MAAD curves that are missing their q grid or disagree with its length', async () => { + const row = { + router: 'r1', + bucketStart: 100, + bucketEnd: 200, + saTau: f32([0.2]), + saTauSd: f32([0.01]), + daTau: null, + daTauSd: null + }; + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + const url = new URL( + 'http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=200' + ); + + for (const qGrid of [undefined, DEFAULT_Q_GRID]) { + mockDatasetSession({ + all: vi.fn().mockResolvedValueOnce([row]).mockResolvedValueOnce([]), + get: vi.fn().mockResolvedValue(qGrid) + }); + const response = await getStructureStats({ url } as never); + expect(response.status).toBe(500); + await expect(response.json()).resolves.toEqual({ error: 'Database query failed' }); + } + + mockDatasetSession({ + all: vi + .fn() + .mockResolvedValueOnce([{ ...row, saTau: null, saTauSd: null }]) + .mockResolvedValueOnce([]), + get: vi.fn().mockResolvedValue(undefined) + }); + const withoutMaad = await getStructureStats({ url } as never); + expect(withoutMaad.status).toBe(200); + }); + + it('fails MAAD spectra that do not hold whole alpha/f pairs', async () => { + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ + all: vi + .fn() + .mockResolvedValueOnce([ + { + router: 'r1', + bucketStart: 100, + bucketEnd: 200, + saSpectrum: f32([1, 2, 3]), + daSpectrum: null + } + ]) + .mockResolvedValueOnce([]) + }); + + const response = await getSpectrumStats({ + url: new URL( + 'http://localhost/api/netflow/spectrum-stats?routers=r1&startDate=100&endDate=200' + ) + } as never); + expect(response.status).toBe(500); + }); + it('keeps selected spectrum sources separate', async () => { const all = vi.fn().mockResolvedValue([]); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); @@ -447,7 +532,7 @@ } as never); expect(response.status).toBe(200); - expect(all).toHaveBeenCalledWith(expect.stringContaining('FROM address_structure_stats'), [ + expect(all).toHaveBeenCalledWith(expect.stringContaining('FROM address_maad_stats'), [ '1h', 'cc_ir1_gw', 'oh_ir1_gw', @@ -501,8 +586,9 @@ it('binds the requested MAAD ip version for structure and spectrum stats', async () => { const all = vi.fn().mockResolvedValue([]); + const get = vi.fn().mockResolvedValue(DEFAULT_Q_GRID); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); - mockDatasetSession({ all }); + mockDatasetSession({ all, get }); const structureResponse = await getStructureStats({ url: new URL( @@ -565,8 +651,9 @@ it('filters structure stats by the requested MAAD measure', async () => { const all = vi.fn().mockResolvedValue([]); + const get = vi.fn().mockResolvedValue(DEFAULT_Q_GRID); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); - mockDatasetSession({ all }); + mockDatasetSession({ all, get }); for (const measure of ['addresses', 'packets', 'bytes']) { all.mockClear(); ``` ## apps/web/tests/routes/netflow-file-routes.test.ts Status: modified; +52; −27 ```diff @@ -23,6 +23,10 @@ ); } +function f32(values: number[]): Buffer { + return Buffer.from(Float32Array.from(values).buffer); +} + describe('netflow file helpers and routes', () => { it('parses slugs and resolves dataset from requests', async () => { vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); @@ -57,11 +61,13 @@ daIpv6Count: 4 }) .mockResolvedValueOnce({ - valuesJson: '[{"alpha":3,"f":4}]' + spectrum: f32([3, 4]) }) .mockResolvedValueOnce({ + tau: f32([2]), + tauSd: f32([0.5]) - valuesJson: '[{"q":1,"tauTilde":2,"sd":0.5}]' - }); + }) + .mockResolvedValueOnce({ ipVersion: 4, qMin: 1, qStep: 1, qCount: 1 }); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); mockDatasetSession({ get @@ -119,6 +125,7 @@ 'source', 'addresses' ]); + expect(get).toHaveBeenNthCalledWith(4, expect.any(String), [4]); await expect(ipResponse.json()).resolves.toEqual({ ipv4Count: 1, ipv6Count: 3 }); await expect(spectrumResponse.json()).resolves.toEqual({ slug: '202503010005', @@ -183,15 +190,19 @@ daIpv4Count: 3, saIpv6Count: 4, daIpv6Count: 5, - structureJsonSa: '[{"q":2,"tauTilde":8,"sd":0.25}]', - structureJsonDa: null, + saTau: f32([8]), + saTauSd: f32([0.25]), + daTau: null, - spectrumJsonSa: '[{"alpha":1.5,"f":2}]', - spectrumJsonDa: null + daTauSd: null, + saSpectrum: f32([1.5, 2]), + daSpectrum: null } ]); + const get = vi.fn().mockResolvedValue({ ipVersion: 4, qMin: 2, qStep: 1, qCount: 1 }); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); mockDatasetSession({ - all + all, + get }); const response = await getDetails({ @@ -216,12 +227,6 @@ 'internal', 'external', 'addresses', - '5m', - bucketStart, - 4, - 'internal', - 'external', - 'addresses', bucketStart ]); await expect(response.json()).resolves.toEqual({ @@ -309,11 +314,31 @@ expect(withDatasetDb).not.toHaveBeenCalledWith('alpha', undefined, expect.any(Function)); }); + it('fails a per-file structure function stored without a matching q grid', async () => { + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + const url = new URL('http://localhost/api/netflow/files/x/structure?router=r1&source=true'); + + for (const qGrid of [undefined, { ipVersion: 4, qMin: 1, qStep: 1, qCount: 33 }]) { + mockDatasetSession({ + get: vi + .fn() + .mockResolvedValueOnce({ tau: f32([2]), tauSd: f32([0.5]) }) + .mockResolvedValueOnce(qGrid) + }); + const response = await getStructure({ params: { slug: '202503010005' }, url } as never); + expect(response.status).toBe(500); + await expect(response.json()).resolves.toEqual({ + error: 'Failed to get structure statistics' + }); + } + }); + it('binds the requested MAAD ip version for the per-file spectrum and structure routes', async () => { const get = vi .fn() + .mockResolvedValueOnce({ spectrum: f32([3, 4]) }) - .mockResolvedValueOnce({ valuesJson: '[{"alpha":3,"f":4}]' }) - .mockResolvedValueOnce({ valuesJson: '[{"q":1,"tauTilde":2,"sd":0.5}]' }); + .mockResolvedValueOnce({ tau: f32([2]), tauSd: f32([0.5]) }) + .mockResolvedValueOnce({ ipVersion: 6, qMin: 1, qStep: 1, qCount: 1 }); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); mockDatasetSession({ get }); @@ -352,6 +377,7 @@ 'source', 'addresses' ]); + expect(get).toHaveBeenNthCalledWith(3, expect.any(String), [6]); }); it('rejects an invalid MAAD ip version for the per-file spectrum, structure, and details routes', async () => { @@ -383,7 +409,11 @@ }); it('binds the requested MAAD measure into the per-file and details MAAD queries', async () => { + const get = vi + .fn() + .mockResolvedValueOnce({ tau: f32([2]), tauSd: f32([0.5]) }) + .mockResolvedValueOnce({ ipVersion: 4, qMin: 1, qStep: 1, qCount: 1 }) + .mockResolvedValue({ ipVersion: 4, qMin: 1, qStep: 1, qCount: 1 }); - const get = vi.fn().mockResolvedValue({ valuesJson: '[{"q":1,"tauTilde":2,"sd":0.5}]' }); const all = vi.fn().mockResolvedValue([]); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); mockDatasetSession({ get, all }); @@ -415,21 +445,22 @@ ) } as never); expect(spectrumResponse.status).toBe(400); - expect(get).toHaveBeenCalledTimes(1); + expect(get).toHaveBeenCalledTimes(2); await getDetails({ params: { slug: '202503010005' }, url: new URL('http://localhost/api/netflow/files/x/details?dataset=alpha&measure=bytes') } as never); const [detailsQuery, detailsParams] = all.mock.calls[0] as [string, unknown[]]; + expect(detailsQuery.match(/AND measure = \?/g)).toHaveLength(1); + expect(detailsParams.filter((value) => value === 'bytes')).toHaveLength(1); - expect(detailsQuery.match(/AND measure = \?/g)).toHaveLength(2); - expect(detailsParams.filter((value) => value === 'bytes')).toHaveLength(2); }); it('binds the requested MAAD ip version into the file details MAAD query', async () => { const all = vi.fn().mockResolvedValue([]); + const get = vi.fn().mockResolvedValue({ ipVersion: 6, qMin: -0.5, qStep: 0.125, qCount: 33 }); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all, get }); - mockDatasetSession({ all }); const bucketStart = slugToBucketStart('202503010005'); @@ -454,12 +485,6 @@ 'all', 'all', 'addresses', - '5m', - bucketStart, - 6, - 'all', - 'all', - 'addresses', bucketStart ]); }); ``` ## docs/code/pipeline-contract.md Status: modified; +25; −2 ```diff @@ -87,8 +87,31 @@ granularity. `addresses` weighs each address as 1 and stores structure, spectrum, and dimensions. `packets` and `bytes` weight moments and D1 entropy by the summed counter and store structure and dimensions only. Prefix validity always uses distinct-address counts. A weighted measure excludes +addresses whose counter sums to 0 and records the excluded count in `zero_weight_addrs`. The +measure set is part of the result configuration identity. + +## MAAD storage + +`address_maad_stats` holds one row per bucket, scope, address side, and measure. Dimensions are +`REAL`. The structure function (`tau`), its standard deviation (`tau_sd`), and the spectrum are +little-endian f32 BLOBs. The only rounding is `value as f32`, which rounds to the nearest even. +Nothing else rounds, truncates, or drops a value. + +- `tau` and `tau_sd` hold `q_count` values each. Element `i` is at `q_min + i * q_step` from the + `maad_q_grid` row of the same IP version. +- `spectrum` holds interleaved `(alpha, f)` pairs. A zero-length BLOB is a computed, empty + spectrum. `NULL` means the measure does not compute one. +- A result below the minimum address count stores `NULL` dimensions and curves. +- D0 is `-tau(0)` and D2 is `tau(2)`. Their standard deviations are the `tau_sd` values at those q. + D1 has no standard deviation. + +The schema rejects BLOBs that are not whole f32 values or whole spectrum pairs, and a `tau_sd` +whose length differs from `tau`. `verify` checks every curve against the q grid and the spectrum +presence of every measure. The dashboard API fails a request whose curve lacks a grid or disagrees -addresses whose counter sums to 0 and records the excluded count as `zeroWeightAddrs` in the row +with `q_count` instead of drawing a partial curve. `compare` checks dimensions and each curve -metadata. The measure set is part of the result configuration identity. +element within the MAAD tolerance, and requires identical `maad_q_grid` rows for every IP version +whose curves both databases store in the window. Changing this encoding is a MAAD contract change, +so it needs a fresh product database. Native and CSV ingestion drop flows whose reported packet count is 0 before any statistic accumulates, so they add no traffic, protocols, ports, or addresses. The bucket keeps its observed ``` ## docs/user/operations.md Status: modified; +2; −2 ```diff @@ -36,9 +36,9 @@ `--start` and `--end` are half-open local date or time boundaries. The default timezone is `America/Los_Angeles`. -Scalar values must match exactly. MAAD JSON values compare with an absolute tolerance. The default tolerance is `1e-10` and `--maad-absolute-tolerance` changes it. +Scalar values must match exactly. MAAD dimensions and each element of the stored 32-bit float curves compare with an absolute tolerance. The default tolerance is `1e-10` and `--maad-absolute-tolerance` changes it. Builds whose MAAD values differ slightly can round to adjacent 32-bit floats, so use a tolerance above one 32-bit float step of the largest curve value (about `1e-6`) for those comparisons. -A missing reference row or a shared-value mismatch returns a nonzero exit status. +A missing reference row or a shared-value mismatch returns a nonzero exit status. So does a q grid that is missing or differs for an IP version whose MAAD curves both databases store in the window; `maad_q_grid.mismatched_ip_versions` in the report lists them. ## Publish a local SQLite database ``` [Next page](https://git.cafe/flambo/atlantis/pull/119/diffs?after=ImRvY3MvdXNlci9vcGVyYXRpb25zLm1kIg&expectedVersion=0&format=markdown)