# Files changed · PR \#112 · flambo/atlantis [View on GitCafe](https://git.cafe/flambo/atlantis/pull/112/diffs) Repository: [flambo/atlantis](https://git.cafe/flambo/atlantis) Visibility: public [feat: store packet\- and byte-weighted MAAD measures](https://git.cafe/flambo/atlantis/pull/112.md) Version: 0 Head commit: da1fe86c996136d7df3ccb81325523159a618144 Base commit: b7aff01ad8acc14e1a927e3961e42e320c2c34b7 Comparison base: b7aff01ad8acc14e1a927e3961e42e320c2c34b7 Showing up to 20 entries from this revision. If the pull request changes, reopen the first page before continuing. ## apps/web/drizzle/0005_maad_measures.sql Status: added; +29; −0 ```diff @@ -1,0 +1,29 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_address_structure_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `address_side` text NOT NULL, + `measure` text NOT NULL, + `structure_kind` text NOT NULL, + `values_json` text NOT NULL, + `metadata_json` text NOT NULL, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`, `address_side`, `measure`, `structure_kind`), + CONSTRAINT "address_structure_stats_ip_version_check" CHECK("__new_address_structure_stats"."ip_version" IN (4, 6)), + CONSTRAINT "address_structure_stats_src_locality_check" CHECK("__new_address_structure_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_structure_stats_dst_locality_check" CHECK("__new_address_structure_stats"."dst_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_structure_stats_measure_check" CHECK("__new_address_structure_stats"."measure" IN ('addresses', 'packets', 'bytes')), + CONSTRAINT "address_structure_stats_measure_spectrum_check" CHECK("__new_address_structure_stats"."measure" = 'addresses' OR "__new_address_structure_stats"."structure_kind" <> 'spectrum') +); +--> statement-breakpoint +INSERT INTO `__new_address_structure_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", "measure", "structure_kind", "values_json", "metadata_json", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", 'addresses', "structure_kind", "values_json", "metadata_json", "processed_at" FROM `address_structure_stats`;--> statement-breakpoint +DROP TABLE `address_structure_stats`;--> statement-breakpoint +ALTER TABLE `__new_address_structure_stats` RENAME TO `address_structure_stats`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_query` ON `address_structure_stats` (`granularity`,`bucket_start`,`source_id`,`ip_version`,`src_locality`,`dst_locality`,`address_side`,`measure`,`structure_kind`);--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_timeseries` ON `address_structure_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`ip_version`,`measure`,`structure_kind`,`bucket_start`); ``` ## apps/web/drizzle/meta/0005_snapshot.json Status: added; +1106; −0 ```diff @@ -1,0 +1,1106 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "346de28f-0cfb-4986-8daf-28235ade1fdf", + "prevId": "4a3c1a73-bfe4-4e21-95c9-7ea8a091979f", + "tables": { + "address_count_stats": { + "name": "address_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_address_count": { + "name": "unique_address_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_count_stats_query": { + "name": "idx_address_count_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "isUnique": false + }, + "idx_address_count_stats_timeseries": { + "name": "idx_address_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "name": "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_count_stats_ip_version_check": { + "name": "address_count_stats_ip_version_check", + "value": "\"address_count_stats\".\"ip_version\" IN (4, 6)" + }, + "address_count_stats_src_locality_check": { + "name": "address_count_stats_src_locality_check", + "value": "\"address_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_count_stats_dst_locality_check": { + "name": "address_count_stats_dst_locality_check", + "value": "\"address_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "address_structure_stats": { + "name": "address_structure_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "measure": { + "name": "measure", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "structure_kind": { + "name": "structure_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "values_json": { + "name": "values_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_structure_stats_query": { + "name": "idx_address_structure_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "measure", + "structure_kind" + ], + "isUnique": false + }, + "idx_address_structure_stats_timeseries": { + "name": "idx_address_structure_stats_timeseries", + "columns": [ + "source_id", + "granularity", + "src_locality", + "dst_locality", + "ip_version", + "measure", + "structure_kind", + "bucket_start" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_measure_structure_kind_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "measure", + "structure_kind" + ], + "name": "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_measure_structure_kind_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_structure_stats_ip_version_check": { + "name": "address_structure_stats_ip_version_check", + "value": "\"address_structure_stats\".\"ip_version\" IN (4, 6)" + }, + "address_structure_stats_src_locality_check": { + "name": "address_structure_stats_src_locality_check", + "value": "\"address_structure_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_dst_locality_check": { + "name": "address_structure_stats_dst_locality_check", + "value": "\"address_structure_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_measure_check": { + "name": "address_structure_stats_measure_check", + "value": "\"address_structure_stats\".\"measure\" IN ('addresses', 'packets', 'bytes')" + }, + "address_structure_stats_measure_spectrum_check": { + "name": "address_structure_stats_measure_spectrum_check", + "value": "\"address_structure_stats\".\"measure\" = 'addresses' OR \"address_structure_stats\".\"structure_kind\" <> 'spectrum'" + } + } + }, + "bucket_coverage": { + "name": "bucket_coverage", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "coverage_state": { + "name": "coverage_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "observed_units": { + "name": "observed_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expected_units": { + "name": "expected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rejected_units": { + "name": "rejected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_bucket_coverage_query": { + "name": "idx_bucket_coverage_query", + "columns": ["granularity", "bucket_start", "source_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bucket_coverage_source_id_granularity_bucket_start_pk": { + "columns": ["source_id", "granularity", "bucket_start"], + "name": "bucket_coverage_source_id_granularity_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "bucket_coverage_interval_check": { + "name": "bucket_coverage_interval_check", + "value": "\"bucket_coverage\".\"bucket_end\" > \"bucket_coverage\".\"bucket_start\"" + }, + "bucket_coverage_expected_check": { + "name": "bucket_coverage_expected_check", + "value": "\"bucket_coverage\".\"expected_units\" > 0" + }, + "bucket_coverage_observed_check": { + "name": "bucket_coverage_observed_check", + "value": "\"bucket_coverage\".\"observed_units\" >= 0 AND \"bucket_coverage\".\"observed_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_rejected_check": { + "name": "bucket_coverage_rejected_check", + "value": "\"bucket_coverage\".\"rejected_units\" >= 0 AND \"bucket_coverage\".\"rejected_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_state_check": { + "name": "bucket_coverage_state_check", + "value": "(\"bucket_coverage\".\"coverage_state\" = 'complete' AND \"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'unknown' AND \"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'partial' AND NOT (\"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) AND NOT (\"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0))" + } + } + }, + "datasets": { + "name": "datasets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_start_date": { + "name": "default_start_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_mode": { + "name": "source_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "discovery_mode": { + "name": "discovery_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_locality": { + "name": "has_locality", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_count_stats": { + "name": "port_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_side": { + "name": "port_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_range": { + "name": "port_range", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_port_count": { + "name": "unique_port_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_port_count_stats_timeseries": { + "name": "idx_port_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "port_side", + "port_range" + ], + "name": "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "port_count_stats_ip_version_check": { + "name": "port_count_stats_ip_version_check", + "value": "\"port_count_stats\".\"ip_version\" IN (4, 6)" + }, + "port_count_stats_src_locality_check": { + "name": "port_count_stats_src_locality_check", + "value": "\"port_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "port_count_stats_dst_locality_check": { + "name": "port_count_stats_dst_locality_check", + "value": "\"port_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "processed_inputs": { + "name": "processed_inputs", + "columns": { + "input_kind": { + "name": "input_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_locator": { + "name": "input_locator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "discovered_at": { + "name": "discovered_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_processed_inputs_source_bucket": { + "name": "idx_processed_inputs_source_bucket", + "columns": ["source_id", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk": { + "columns": ["input_kind", "input_locator", "source_id", "bucket_start"], + "name": "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "protocol_stats": { + "name": "protocol_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_protocols_count": { + "name": "unique_protocols_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocols_list": { + "name": "protocols_list", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_protocol_stats_timeseries": { + "name": "idx_protocol_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "protocol_stats_ip_version_check": { + "name": "protocol_stats_ip_version_check", + "value": "\"protocol_stats\".\"ip_version\" IN (4, 6)" + }, + "protocol_stats_src_locality_check": { + "name": "protocol_stats_src_locality_check", + "value": "\"protocol_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "protocol_stats_dst_locality_check": { + "name": "protocol_stats_dst_locality_check", + "value": "\"protocol_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "source_members": { + "name": "source_members", + "columns": { + "dataset_id": { + "name": "dataset_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "source_members_dataset_id_source_id_member_id_pk": { + "columns": ["dataset_id", "source_id", "member_id"], + "name": "source_members_dataset_id_source_id_member_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "traffic_stats": { + "name": "traffic_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows": { + "name": "flows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_tcp": { + "name": "flows_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_udp": { + "name": "flows_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_icmp": { + "name": "flows_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_other": { + "name": "flows_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets": { + "name": "packets", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_tcp": { + "name": "packets_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_udp": { + "name": "packets_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_icmp": { + "name": "packets_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_other": { + "name": "packets_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_tcp": { + "name": "bytes_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_udp": { + "name": "bytes_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_icmp": { + "name": "bytes_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_other": { + "name": "bytes_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_sum_ms": { + "name": "duration_sum_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_count": { + "name": "duration_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_duration_ms": { + "name": "average_duration_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_ttl_sum": { + "name": "min_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_ttl_count": { + "name": "min_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_min_ttl": { + "name": "average_min_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_ttl_sum": { + "name": "max_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_ttl_count": { + "name": "max_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_max_ttl": { + "name": "average_max_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_traffic_stats_query": { + "name": "idx_traffic_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality" + ], + "isUnique": false + }, + "idx_traffic_stats_timeseries": { + "name": "idx_traffic_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "traffic_stats_ip_version_check": { + "name": "traffic_stats_ip_version_check", + "value": "\"traffic_stats\".\"ip_version\" IN (4, 6)" + }, + "traffic_stats_src_locality_check": { + "name": "traffic_stats_src_locality_check", + "value": "\"traffic_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "traffic_stats_dst_locality_check": { + "name": "traffic_stats_dst_locality_check", + "value": "\"traffic_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} ``` ## apps/web/drizzle/meta/\_journal.json Status: modified; +7; −0 ```diff @@ -36,6 +36,13 @@ "when": 1790313292395, "tag": "0004_dataset_locality", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1790328400092, + "tag": "0005_maad_measures", + "breakpoints": true } ] } ``` ## apps/web/src/lib/components/filters/MaadMeasureFilter.svelte Status: added; +31; −0 ```diff @@ -1,0 +1,31 @@ + + + ``` ## apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte Status: modified; +41; −15 ```diff @@ -6,11 +6,15 @@ import NetflowFileMessageCard from '$lib/components/netflow/NetflowFileMessageCard.svelte'; import NetflowFileRouterCard from '$lib/components/netflow/NetflowFileRouterCard.svelte'; import SegmentedControl from '$lib/components/common/SegmentedControl.svelte'; + import MaadMeasureFilter from '$lib/components/filters/MaadMeasureFilter.svelte'; import { navigateToNetflowFile } from '$lib/utils/netflow-file-navigation'; import { + DEFAULT_MAAD_MEASURE, MAAD_IP_VERSION_OPTIONS, + maadMeasureHasSpectrum, type FlowDirection, + type MaadIpVersion, - type MaadIpVersion + type MaadMeasure } from '$lib/types/types'; import { createDateFromPSTComponents, @@ -36,6 +40,8 @@ let { data }: { data: NetflowFileDetailData } = $props(); let loader = $state.raw | null>(null); + let maadMeasure = $state(DEFAULT_MAAD_MEASURE); + const showSpectrum = $derived(maadMeasureHasSpectrum(maadMeasure)); const formatCount = (value: number | null | undefined) => typeof value === 'number' && Number.isFinite(value) ? value.toLocaleString() : 'N/A'; @@ -59,7 +65,13 @@ const nextSlug = $derived(getNextSlug(data.slug)); function syncLoader() { + loader = getNetflowFileDetailLoader( + data.dataset, + data.slug, + data.direction, + data.ipVersion, - loader = getNetflowFileDetailLoader(data.dataset, data.slug, data.direction, data.ipVersion); + maadMeasure + ); loader.refresh(); } @@ -78,6 +90,14 @@ }); } + function handleMaadMeasureChange({ measure }: { measure: MaadMeasure }) { + if (measure === maadMeasure) { + return; + } + maadMeasure = measure; + syncLoader(); + } + onMount(() => { syncLoader(); }); @@ -106,18 +126,24 @@ : 'N/A'} /> -
- MAAD address family: - ({ - value: String(option.value), - label: option.label - }))} - value={String(data.ipVersion)} - onValueChange={(value) => handleMaadIpVersionChange(Number(value) as MaadIpVersion)} - ariaLabel="Select MAAD IP address family" - buttonClass="sm:min-w-32" +
+
+ MAAD address family: + ({ + value: String(option.value), - /> + label: option.label + }))} + value={String(data.ipVersion)} + onValueChange={(value) => handleMaadIpVersionChange(Number(value) as MaadIpVersion)} + ariaLabel="Select MAAD IP address family" + buttonClass="sm:min-w-32" + /> +
+
+ MAAD measure: + +
{#if loader?.error && !loader.hasRows} @@ -144,7 +170,7 @@ /> {/if} {#each loader.rows as row (row.key)} + - {/each}
{/if} ``` ## apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte Status: modified; +14; −6 ```diff @@ -6,10 +6,12 @@ let { row, + showSpectrum = true, formatCount, formatTimestampAsPST }: { row: NetflowFileRouterRow; + showSpectrum?: boolean; formatCount: (value: number | null | undefined) => string; formatTimestampAsPST: (timestamp: number) => string; } = $props(); @@ -35,12 +37,18 @@ source={row.source.structure} destination={row.destination.structure} /> - + title="Spectrum" + kind="spectrum" + source={row.source.spectrum} + destination={row.destination.spectrum} + /> + {:else} +

+ The spectrum is only computed for the addresses measure. +

+ {/if} ``` ## apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts Status: modified; +27; −10 ```diff @@ -1,8 +1,10 @@ import { DEFAULT_MAAD_IP_VERSION, + DEFAULT_MAAD_MEASURE, type FileIpCounts, type FlowDirection, type MaadIpVersion, + type MaadMeasure, type NetflowFileDetailsResponse, type NetflowFileSummaryRecord, type SpectrumData, @@ -72,14 +74,16 @@ dataset: string, slug: string, direction: FlowDirection, + ipVersion: MaadIpVersion, - ipVersion: MaadIpVersion + measure: MaadMeasure ): string { - return `${dataset}:${slug}:${direction}:${ipVersion}`; + return `${dataset}:${slug}:${direction}:${ipVersion}:${measure}`; } function buildScopeSearchParams( direction: FlowDirection, + ipVersion: MaadIpVersion, - ipVersion: MaadIpVersion + measure: MaadMeasure ): Record { const params: Record = {}; @@ -91,6 +95,10 @@ params.ipVersion = String(ipVersion); } + if (measure !== DEFAULT_MAAD_MEASURE) { + params.measure = measure; + } + return params; } @@ -112,6 +120,7 @@ readonly slug: string; readonly direction: FlowDirection; readonly ipVersion: MaadIpVersion; + readonly measure: MaadMeasure; readonly key: string; private _loading = $state(false); @@ -124,12 +133,19 @@ private _lastAccessedAt = Date.now(); private summarySkeletonDelayId: ReturnType | null = null; + constructor( + dataset: string, + slug: string, + direction: FlowDirection, + ipVersion: MaadIpVersion, + measure: MaadMeasure + ) { - constructor(dataset: string, slug: string, direction: FlowDirection, ipVersion: MaadIpVersion) { this.dataset = dataset; this.slug = slug; this.direction = direction; this.ipVersion = ipVersion; + this.measure = measure; - this.key = buildLoaderKey(dataset, slug, direction, ipVersion); + this.key = buildLoaderKey(dataset, slug, direction, ipVersion, measure); } get loading() { @@ -299,7 +315,7 @@ const response = await fetch( `/api/netflow/files/${this.slug}/details?${new URLSearchParams({ dataset: this.dataset, - ...buildScopeSearchParams(this.direction, this.ipVersion) + ...buildScopeSearchParams(this.direction, this.ipVersion, this.measure) }).toString()}` ); @@ -379,7 +395,7 @@ return `/api/netflow/files/${this.slug}/${kind === 'ipCounts' ? 'ip-counts' : kind}?${new URLSearchParams( { dataset: this.dataset, - ...buildScopeSearchParams(this.direction, this.ipVersion), + ...buildScopeSearchParams(this.direction, this.ipVersion, this.measure), router, source: String(side === 'source') } @@ -412,13 +428,14 @@ dataset: string, slug: string, direction: FlowDirection, + ipVersion: MaadIpVersion = DEFAULT_MAAD_IP_VERSION, - ipVersion: MaadIpVersion = DEFAULT_MAAD_IP_VERSION + measure: MaadMeasure = DEFAULT_MAAD_MEASURE ) { - const key = buildLoaderKey(dataset, slug, direction, ipVersion); + const key = buildLoaderKey(dataset, slug, direction, ipVersion, measure); let loader = fileDetailLoaders.get(key); if (!loader) { - loader = new NetflowFileDetailLoader(dataset, slug, direction, ipVersion); + loader = new NetflowFileDetailLoader(dataset, slug, direction, ipVersion, measure); fileDetailLoaders.set(key, loader); } ``` ## apps/web/src/lib/server/db/local-schema.ts Status: modified; +5; −3 ```diff @@ -121,13 +121,15 @@ src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), address_side TEXT NOT NULL CHECK(address_side IN ('source', 'destination')), + measure TEXT NOT NULL CHECK(measure IN ('addresses', 'packets', 'bytes')), structure_kind TEXT NOT NULL CHECK(structure_kind IN ('structure', 'spectrum', 'dimension')), values_json TEXT NOT NULL, metadata_json TEXT NOT NULL, processed_at TEXT DEFAULT CURRENT_TIMESTAMP, + CHECK(measure = 'addresses' OR structure_kind <> 'spectrum'), PRIMARY KEY( source_id, granularity, bucket_start, ip_version, - src_locality, dst_locality, address_side, structure_kind + src_locality, dst_locality, address_side, measure, structure_kind ) ); @@ -181,12 +183,12 @@ CREATE INDEX IF NOT EXISTS idx_address_structure_stats_query ON address_structure_stats ( granularity, bucket_start, source_id, ip_version, - src_locality, dst_locality, address_side, structure_kind + src_locality, dst_locality, address_side, measure, structure_kind ); CREATE INDEX IF NOT EXISTS idx_address_structure_stats_timeseries ON address_structure_stats ( source_id, granularity, src_locality, dst_locality, - ip_version, structure_kind, bucket_start + ip_version, measure, structure_kind, bucket_start ); CREATE INDEX IF NOT EXISTS idx_port_count_stats_timeseries ON port_count_stats ( ``` ## apps/web/src/lib/server/db/schema.ts Status: modified; +12; −0 ```diff @@ -331,6 +331,7 @@ srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), addressSide: text('address_side', { enum: ['source', 'destination'] }).notNull(), + measure: text('measure', { enum: ['addresses', 'packets', 'bytes'] }).notNull(), structureKind: text('structure_kind', { enum: ['structure', 'spectrum', 'dimension'] }).notNull(), @@ -348,6 +349,7 @@ table.srcLocality, table.dstLocality, table.addressSide, + table.measure, table.structureKind ] }), @@ -359,6 +361,7 @@ table.srcLocality, table.dstLocality, table.addressSide, + table.measure, table.structureKind ), index('idx_address_structure_stats_timeseries').on( @@ -367,6 +370,7 @@ table.srcLocality, table.dstLocality, table.ipVersion, + table.measure, table.structureKind, table.bucketStart ), @@ -378,6 +382,14 @@ check( 'address_structure_stats_dst_locality_check', sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ), + check( + 'address_structure_stats_measure_check', + sql`${table.measure} IN ('addresses', 'packets', 'bytes')` + ), + check( + 'address_structure_stats_measure_spectrum_check', + sql`${table.measure} = 'addresses' OR ${table.structureKind} <> 'spectrum'` ) ] ); ``` ## apps/web/src/lib/server/netflow-v3.ts Status: modified; +62; −1 ```diff @@ -1,14 +1,17 @@ import { DEFAULT_MAAD_IP_VERSION, + DEFAULT_MAAD_MEASURE, FLOW_DIRECTIONS, flowDirectionLocalities, IP_GRANULARITIES, MAAD_IP_VERSIONS, + MAAD_MEASURES, type FlowDirection, type FlowLocality, type FlowLocalityPair, type IpGranularity, - type MaadIpVersion + type MaadIpVersion, + type MaadMeasure } from '$lib/types/types'; import type { SourceDefinition } from '$lib/server/datasets'; import type { StructureFunctionPoint } from '$lib/types/types'; @@ -30,6 +33,13 @@ dstLocality: FlowLocality; } +export interface MaadParams { + ipVersion: MaadIpVersion; + measure: MaadMeasure; +} + +export type MaadStatsParams = AggregateStatsParams & MaadParams; + export interface RequestValidationError { error: string; status: 400; @@ -41,6 +51,7 @@ const VALID_IP_GRANULARITIES = new Set(IP_GRANULARITIES); const VALID_FLOW_DIRECTIONS = new Set(FLOW_DIRECTIONS); +const VALID_MAAD_MEASURES = new Set(MAAD_MEASURES); export function assertNetflowV3Database(): void { return; @@ -117,6 +128,56 @@ return version; } +export function parseMaadMeasure(url: URL): MaadMeasure | RequestValidationError { + const param = url.searchParams.get('measure'); + if (param === null) { + return DEFAULT_MAAD_MEASURE; + } + + if (!VALID_MAAD_MEASURES.has(param)) { + return { + error: `Invalid measure. Expected one of: ${MAAD_MEASURES.join(', ')}`, + status: 400 + }; + } + + return param as MaadMeasure; +} + +export function parseMaadParams(url: URL): MaadParams | RequestValidationError { + const ipVersion = parseMaadIpVersion(url); + if (typeof ipVersion !== 'number') { + return ipVersion; + } + + const measure = parseMaadMeasure(url); + if (typeof measure !== 'string') { + return measure; + } + + return { ipVersion, measure }; +} + +export function parseMaadStatsParams(url: URL): MaadStatsParams | RequestValidationError { + const aggregate = parseAggregateStatsParams(url); + if ('error' in aggregate) { + return aggregate; + } + + const maad = parseMaadParams(url); + if ('error' in maad) { + return maad; + } + + return { ...aggregate, ...maad }; +} + +export function spectrumMeasureError(measure: MaadMeasure): RequestValidationError | null { + return measure === 'addresses' + ? null + : { error: `Spectrum is only computed for the addresses measure, not ${measure}`, status: 400 }; +} + export function parseFlowDirectionParams( url: URL ): ({ direction: FlowDirection } & FlowLocalityPair) | RequestValidationError { ``` ## apps/web/src/lib/types/types.ts Status: modified; +16; −0 ```diff @@ -378,7 +378,23 @@ { value: 4, label: 'IPv4 (/8–/24)' }, { value: 6, label: 'IPv6 (/23–/64)' } ]; + +export const MAAD_MEASURES = ['addresses', 'packets', 'bytes'] as const; + +export type MaadMeasure = (typeof MAAD_MEASURES)[number]; +export const DEFAULT_MAAD_MEASURE: MaadMeasure = 'addresses'; + +export const MAAD_MEASURE_OPTIONS: { value: MaadMeasure; label: string }[] = [ + { value: 'addresses', label: 'Addresses' }, + { value: 'packets', label: 'Packets' }, + { value: 'bytes', label: 'Bytes' } +]; + +export function maadMeasureHasSpectrum(measure: MaadMeasure): boolean { + return measure === 'addresses'; +} + export interface SpectrumPoint { alpha: number; f: number; ``` ## apps/web/src/routes/api/netflow/files/\[slug\]/details/\+server.ts Status: modified; +10; −6 ```diff @@ -14,7 +14,7 @@ import { normalizeStructurePoints, parseFlowDirectionParams, - parseMaadIpVersion + parseMaadParams } from '$lib/server/netflow-v3'; const FIVE_MINUTES = '5m'; @@ -133,9 +133,9 @@ return json({ error: flowDirection.error }, { status: flowDirection.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -210,6 +210,7 @@ AND ip_version = ? AND src_locality = ? AND dst_locality = ? + AND measure = ? AND structure_kind = 'structure' GROUP BY source_id, bucket_start ), @@ -225,6 +226,7 @@ AND ip_version = ? AND src_locality = ? AND dst_locality = ? + AND measure = ? AND structure_kind = 'spectrum' GROUP BY source_id, bucket_start ) @@ -281,14 +283,16 @@ flowDirection.dstLocality, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + maad.measure, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + maad.measure, bucketStart ] ); ``` ## apps/web/src/routes/api/netflow/files/\[slug\]/spectrum/\+server.ts Status: modified; +17; −6 ```diff @@ -1,7 +1,11 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import type { SpectrumData, SpectrumPoint } from '$lib/types/types'; +import { + parseFlowDirectionParams, -import { parseFlowDirectionParams, parseMaadIpVersion } from '$lib/server/netflow-v3'; + parseMaadParams, + spectrumMeasureError +} from '$lib/server/netflow-v3'; import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; const FIVE_MINUTES = '5m'; @@ -21,9 +25,14 @@ return json({ error: flowDirection.error }, { status: flowDirection.status }); } + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); + } + + const measureError = spectrumMeasureError(maad.measure); - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + if (measureError) { + return json({ error: measureError.error }, { status: measureError.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -61,16 +70,18 @@ AND src_locality = ? AND dst_locality = ? AND address_side = ? + AND measure = ? AND structure_kind = 'spectrum' LIMIT 1`, [ router, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + isSource ? 'source' : 'destination', - isSource ? 'source' : 'destination' + maad.measure ] ); ``` ## apps/web/src/routes/api/netflow/files/\[slug\]/structure/\+server.ts Status: modified; +8; −6 ```diff @@ -5,7 +5,7 @@ import { normalizeStructurePoints, parseFlowDirectionParams, - parseMaadIpVersion + parseMaadParams } from '$lib/server/netflow-v3'; const FIVE_MINUTES = '5m'; @@ -25,9 +25,9 @@ return json({ error: flowDirection.error }, { status: flowDirection.status }); } + const maad = parseMaadParams(url); + if ('error' in maad) { - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + return json({ error: maad.error }, { status: maad.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -65,16 +65,18 @@ AND src_locality = ? AND dst_locality = ? AND address_side = ? + AND measure = ? AND structure_kind = 'structure' LIMIT 1`, [ router, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + isSource ? 'source' : 'destination', + maad.measure - isSource ? 'source' : 'destination' ] ); ``` ## apps/web/src/routes/api/netflow/spectrum-stats/\+server.ts Status: modified; +9; −11 ```diff @@ -4,11 +4,7 @@ import type { SpectrumStatsPayload, SpectrumStatsResponse } from '$lib/types/spectrum-stats'; import { buildCoverageTimelines } from '$lib/server/db/coverage'; import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; -import { - parseAggregateStatsParams, - parseMaadIpVersion, - placeholders -} from '$lib/server/netflow-v3'; +import { parseMaadStatsParams, placeholders, spectrumMeasureError } from '$lib/server/netflow-v3'; type SpectrumStatsRow = SpectrumStatsPayload & { router: string; @@ -53,15 +49,15 @@ } export const GET: RequestHandler = async ({ url, platform }) => { + const params = parseMaadStatsParams(url); - const params = parseAggregateStatsParams(url); if ('error' in params) { return json({ error: params.error }, { status: params.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { + const measureError = spectrumMeasureError(params.measure); + if (measureError) { + return json({ error: measureError.error }, { status: measureError.status }); - return json({ error: ipVersion.error }, { status: ipVersion.status }); } - const { routers, granularity, start, end, srcLocality, dstLocality } = params; + const { routers, granularity, start, end, srcLocality, dstLocality, ipVersion, measure } = params; try { const dataset = await getRequestedDataset(url, platform); @@ -75,7 +71,8 @@ dstLocality, start, end, + ipVersion, + measure - ipVersion ]; const query = ` @@ -93,6 +90,7 @@ AND bucket_start >= ? AND bucket_start < ? AND ip_version = ? + AND measure = ? AND structure_kind = 'spectrum' GROUP BY ${sourceColumn}, bucket_start `; ``` ## apps/web/src/routes/api/netflow/structure-stats/\+server.ts Status: modified; +6; −9 ```diff @@ -6,8 +6,7 @@ import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; import { normalizeStructurePoints, - parseAggregateStatsParams, - parseMaadIpVersion, + parseMaadStatsParams, placeholders } from '$lib/server/netflow-v3'; @@ -64,15 +63,11 @@ } export const GET: RequestHandler = async ({ url, platform }) => { - const params = parseAggregateStatsParams(url); + const params = parseMaadStatsParams(url); if ('error' in params) { return json({ error: params.error }, { status: params.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { + const { routers, granularity, start, end, srcLocality, dstLocality, ipVersion, measure } = params; - return json({ error: ipVersion.error }, { status: ipVersion.status }); - } - const { routers, granularity, start, end, srcLocality, dstLocality } = params; try { const dataset = await getRequestedDataset(url, platform); @@ -86,7 +81,8 @@ dstLocality, start, end, + ipVersion, - ipVersion + measure ]; const query = ` @@ -104,6 +100,7 @@ AND bucket_start >= ? AND bucket_start < ? AND ip_version = ? + AND measure = ? AND structure_kind = 'structure' GROUP BY ${sourceColumn}, bucket_start `; ``` ## apps/web/tests/lib/server/migrations.test.ts Status: modified; +46; −0 ```diff @@ -162,6 +162,7 @@ 'src_locality', 'dst_locality', 'ip_version', + 'measure', 'structure_kind', 'bucket_start' ] @@ -342,4 +343,49 @@ database.close(); } }); + + it('keys MAAD rows by measure and keeps existing rows as the addresses measure', () => { + const database = new Database(':memory:'); + const migrations = migrationFiles(); + const measureMigration = migrations.findIndex((fileName) => + fs + .readFileSync(path.join(migrationsDirectory, fileName), 'utf8') + .includes('address_structure_stats_measure_check') + ); + try { + for (const migration of migrations.slice(0, measureMigration)) { + applyMigration(database, migration); + } + database + .prepare( + `INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, structure_kind, + values_json, metadata_json + ) VALUES ('edge', '5m', 0, 300, 4, 'all', 'all', 'source', 'spectrum', '[]', '{}')` + ) + .run(); + + for (const migration of migrations.slice(measureMigration)) { + applyMigration(database, migration); + } + + expect( + database.prepare('SELECT measure, structure_kind FROM address_structure_stats').all() + ).toEqual([{ measure: 'addresses', structure_kind: 'spectrum' }]); + const insert = database.prepare( + `INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('edge', '5m', 0, 300, 4, 'all', 'all', 'source', ?, ?, '[]', '{}')` + ); + insert.run('packets', 'structure'); + insert.run('bytes', 'structure'); + expect(() => insert.run('packets', 'spectrum')).toThrow(/CHECK constraint failed/); + expect(() => insert.run('flows', 'structure')).toThrow(/CHECK constraint failed/); + } finally { + database.close(); + } + }); }); ``` ## apps/web/tests/lib/server/netflow-v3.test.ts Status: modified; +46; −0 ```diff @@ -9,6 +9,8 @@ parseFlowDirection, parseFlowDirectionParams, parseMaadIpVersion, + parseMaadMeasure, + parseMaadStatsParams, parseSourceIds, parseTimestamp, resolveSourceIds @@ -169,6 +171,50 @@ } }); + it('parses the MAAD measure with an addresses default', () => { + expect(parseMaadMeasure(new URL('http://localhost/api/test'))).toBe('addresses'); + for (const measure of ['addresses', 'packets', 'bytes']) { + expect(parseMaadMeasure(new URL(`http://localhost/api/test?measure=${measure}`))).toBe( + measure + ); + } + expect(parseMaadMeasure(new URL('http://localhost/api/test?measure=Packets'))).toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes', + status: 400 + }); + }); + + it('extends aggregate stats params with the MAAD ip version and measure', () => { + expect( + parseMaadStatsParams( + new URL( + 'http://localhost/api/test?routers=r1&startDate=100&endDate=200&direction=egress&ipVersion=6&measure=bytes' + ) + ) + ).toEqual({ + routers: ['r1'], + granularity: '1h', + start: 100, + end: 200, + direction: 'egress', + srcLocality: 'internal', + dstLocality: 'external', + ipVersion: 6, + measure: 'bytes' + }); + expect( + parseMaadStatsParams( + new URL('http://localhost/api/test?routers=r1&startDate=100&endDate=200&measure=flows') + ) + ).toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes', + status: 400 + }); + expect( + parseMaadStatsParams(new URL('http://localhost/api/test?startDate=100&endDate=200')) + ).toEqual({ error: 'No routers selected', status: 400 }); + }); + it('normalizes structure points from MAAD variants', () => { expect(normalizeStructurePoints([{ q: 1, tauTilde: 2, s: 3 }])).toEqual([ { q: 1, tau: 2, sd: 3 } ``` ## apps/web/tests/performance/seed-large-db.ts Status: modified; +2; −2 ```diff @@ -195,11 +195,11 @@ INSERT INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json ) SELECT source_id, '1h', bucket_start, bucket_start + 3600, 4, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, 'addresses', structure_kind, CASE structure_kind WHEN 'spectrum' THEN '[{"alpha":0.1,"f":0.2},{"alpha":0.2,"f":0.3},{"alpha":0.3,"f":0.4}]' ELSE '[{"q":1,"tau":0.2,"sd":0.01},{"q":2,"tau":0.4,"sd":0.02}]' ``` ## apps/web/tests/routes/api-aggregate-routes.test.ts Status: modified; +59; −3 ```diff @@ -456,7 +456,8 @@ 'all', 100, 200, - 4 + 4, + 'addresses' ]); await expect(response.json()).resolves.toEqual({ timelines: [ @@ -516,7 +517,8 @@ 'all', 100, 200, - 6 + 6, + 'addresses' ]); const spectrumResponse = await getSpectrumStats({ @@ -532,7 +534,8 @@ 'all', 100, 200, - 6 + 6, + 'addresses' ]); }); @@ -559,4 +562,57 @@ error: 'Invalid ipVersion. Expected one of: 4, 6' }); }); + + it('filters structure stats by the requested MAAD measure', async () => { + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all }); + + for (const measure of ['addresses', 'packets', 'bytes']) { + all.mockClear(); + const response = await getStructureStats({ + url: new URL( + `http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=200&measure=${measure}` + ) + } as never); + expect(response.status).toBe(200); + expect(all).toHaveBeenNthCalledWith(1, expect.stringContaining('AND measure = ?'), [ + '1h', + 'r1', + 'all', + 'all', + 100, + 200, + 4, + measure + ]); + } + }); + + it('rejects unknown measures and weighted spectrum requests', async () => { + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all }); + + const invalid = await getStructureStats({ + url: new URL( + 'http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=200&measure=flows' + ) + } as never); + expect(invalid.status).toBe(400); + await expect(invalid.json()).resolves.toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes' + }); + + const weightedSpectrum = await getSpectrumStats({ + url: new URL( + 'http://localhost/api/netflow/spectrum-stats?routers=r1&startDate=100&endDate=200&measure=packets' + ) + } as never); + expect(weightedSpectrum.status).toBe(400); + await expect(weightedSpectrum.json()).resolves.toEqual({ + error: 'Spectrum is only computed for the addresses measure, not packets' + }); + expect(all).not.toHaveBeenCalled(); + }); }); ``` [Next page](https://git.cafe/flambo/atlantis/pull/112/diffs?after=ImFwcHMvd2ViL3Rlc3RzL3JvdXRlcy9hcGktYWdncmVnYXRlLXJvdXRlcy50ZXN0LnRzIg&expectedVersion=0&format=markdown)