# Files changed · PR \#108 · flambo/atlantis [View on GitCafe](https://git.cafe/flambo/atlantis/pull/108/diffs) Repository: [flambo/atlantis](https://git.cafe/flambo/atlantis) Visibility: public [feat: classify endpoint locality and filter by traffic direction](https://git.cafe/flambo/atlantis/pull/108.md) Version: 0 Head commit: fd5f115fdc833e1706341d4846d1d9d7616dc93e Base commit: 478218db51cfbc59d7d0fc4f6353d6ab2d156689 Comparison base: 478218db51cfbc59d7d0fc4f6353d6ab2d156689 Showing up to 20 entries from this revision. If the pull request changes, reopen the first page before continuing. ## apps/web/drizzle/0003_mixed_vin_gonzales.sql Status: added; +135; −0 ```diff @@ -1,0 +1,135 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_address_count_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `address_side` text NOT NULL, + `unique_address_count` integer NOT NULL, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`, `address_side`), + CONSTRAINT "address_count_stats_ip_version_check" CHECK("__new_address_count_stats"."ip_version" IN (4, 6)), + CONSTRAINT "address_count_stats_src_locality_check" CHECK("__new_address_count_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_count_stats_dst_locality_check" CHECK("__new_address_count_stats"."dst_locality" IN ('all', 'internal', 'external')) +); +--> statement-breakpoint +INSERT INTO `__new_address_count_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", "unique_address_count", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_visibility", "dst_visibility", "address_side", "unique_address_count", "processed_at" FROM `address_count_stats` WHERE "src_visibility" = 'all' AND "dst_visibility" = 'all';--> statement-breakpoint +DROP TABLE `address_count_stats`;--> statement-breakpoint +ALTER TABLE `__new_address_count_stats` RENAME TO `address_count_stats`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE INDEX `idx_address_count_stats_query` ON `address_count_stats` (`granularity`,`bucket_start`,`source_id`,`ip_version`,`src_locality`,`dst_locality`,`address_side`);--> statement-breakpoint +CREATE INDEX `idx_address_count_stats_timeseries` ON `address_count_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`bucket_start`);--> statement-breakpoint +CREATE TABLE `__new_address_structure_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `address_side` text NOT NULL, + `structure_kind` text NOT NULL, + `values_json` text NOT NULL, + `metadata_json` text NOT NULL, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`, `address_side`, `structure_kind`), + CONSTRAINT "address_structure_stats_ip_version_check" CHECK("__new_address_structure_stats"."ip_version" IN (4, 6)), + CONSTRAINT "address_structure_stats_src_locality_check" CHECK("__new_address_structure_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_structure_stats_dst_locality_check" CHECK("__new_address_structure_stats"."dst_locality" IN ('all', 'internal', 'external')) +); +--> statement-breakpoint +INSERT INTO `__new_address_structure_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", "structure_kind", "values_json", "metadata_json", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_visibility", "dst_visibility", "address_side", "structure_kind", "values_json", "metadata_json", "processed_at" FROM `address_structure_stats` WHERE "src_visibility" = 'all' AND "dst_visibility" = 'all';--> statement-breakpoint +DROP TABLE `address_structure_stats`;--> statement-breakpoint +ALTER TABLE `__new_address_structure_stats` RENAME TO `address_structure_stats`;--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_query` ON `address_structure_stats` (`granularity`,`bucket_start`,`source_id`,`ip_version`,`src_locality`,`dst_locality`,`address_side`,`structure_kind`);--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_timeseries` ON `address_structure_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`ip_version`,`structure_kind`,`bucket_start`);--> statement-breakpoint +CREATE TABLE `__new_port_count_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `port_side` text NOT NULL, + `port_range` text NOT NULL, + `unique_port_count` integer NOT NULL, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`, `port_side`, `port_range`), + CONSTRAINT "port_count_stats_ip_version_check" CHECK("__new_port_count_stats"."ip_version" IN (4, 6)), + CONSTRAINT "port_count_stats_src_locality_check" CHECK("__new_port_count_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "port_count_stats_dst_locality_check" CHECK("__new_port_count_stats"."dst_locality" IN ('all', 'internal', 'external')) +); +--> statement-breakpoint +INSERT INTO `__new_port_count_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "port_side", "port_range", "unique_port_count", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_visibility", "dst_visibility", "port_side", "port_range", "unique_port_count", "processed_at" FROM `port_count_stats` WHERE "src_visibility" = 'all' AND "dst_visibility" = 'all';--> statement-breakpoint +DROP TABLE `port_count_stats`;--> statement-breakpoint +ALTER TABLE `__new_port_count_stats` RENAME TO `port_count_stats`;--> statement-breakpoint +CREATE INDEX `idx_port_count_stats_timeseries` ON `port_count_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`bucket_start`);--> statement-breakpoint +CREATE TABLE `__new_protocol_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `unique_protocols_count` integer NOT NULL, + `protocols_list` text NOT NULL, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`), + CONSTRAINT "protocol_stats_ip_version_check" CHECK("__new_protocol_stats"."ip_version" IN (4, 6)), + CONSTRAINT "protocol_stats_src_locality_check" CHECK("__new_protocol_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "protocol_stats_dst_locality_check" CHECK("__new_protocol_stats"."dst_locality" IN ('all', 'internal', 'external')) +); +--> statement-breakpoint +INSERT INTO `__new_protocol_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "unique_protocols_count", "protocols_list", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_visibility", "dst_visibility", "unique_protocols_count", "protocols_list", "processed_at" FROM `protocol_stats` WHERE "src_visibility" = 'all' AND "dst_visibility" = 'all';--> statement-breakpoint +DROP TABLE `protocol_stats`;--> statement-breakpoint +ALTER TABLE `__new_protocol_stats` RENAME TO `protocol_stats`;--> statement-breakpoint +CREATE INDEX `idx_protocol_stats_timeseries` ON `protocol_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`bucket_start`);--> statement-breakpoint +CREATE TABLE `__new_traffic_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `flows` integer NOT NULL, + `flows_tcp` integer NOT NULL, + `flows_udp` integer NOT NULL, + `flows_icmp` integer NOT NULL, + `flows_other` integer NOT NULL, + `packets` integer NOT NULL, + `packets_tcp` integer NOT NULL, + `packets_udp` integer NOT NULL, + `packets_icmp` integer NOT NULL, + `packets_other` integer NOT NULL, + `bytes` integer NOT NULL, + `bytes_tcp` integer NOT NULL, + `bytes_udp` integer NOT NULL, + `bytes_icmp` integer NOT NULL, + `bytes_other` integer NOT NULL, + `duration_sum_ms` integer NOT NULL, + `duration_count` integer NOT NULL, + `average_duration_ms` real, + `min_ttl_sum` integer NOT NULL, + `min_ttl_count` integer NOT NULL, + `average_min_ttl` real, + `max_ttl_sum` integer NOT NULL, + `max_ttl_count` integer NOT NULL, + `average_max_ttl` real, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`), + CONSTRAINT "traffic_stats_ip_version_check" CHECK("__new_traffic_stats"."ip_version" IN (4, 6)), + CONSTRAINT "traffic_stats_src_locality_check" CHECK("__new_traffic_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "traffic_stats_dst_locality_check" CHECK("__new_traffic_stats"."dst_locality" IN ('all', 'internal', 'external')) +); +--> statement-breakpoint +INSERT INTO `__new_traffic_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "flows", "flows_tcp", "flows_udp", "flows_icmp", "flows_other", "packets", "packets_tcp", "packets_udp", "packets_icmp", "packets_other", "bytes", "bytes_tcp", "bytes_udp", "bytes_icmp", "bytes_other", "duration_sum_ms", "duration_count", "average_duration_ms", "min_ttl_sum", "min_ttl_count", "average_min_ttl", "max_ttl_sum", "max_ttl_count", "average_max_ttl", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_visibility", "dst_visibility", "flows", "flows_tcp", "flows_udp", "flows_icmp", "flows_other", "packets", "packets_tcp", "packets_udp", "packets_icmp", "packets_other", "bytes", "bytes_tcp", "bytes_udp", "bytes_icmp", "bytes_other", "duration_sum_ms", "duration_count", "average_duration_ms", "min_ttl_sum", "min_ttl_count", "average_min_ttl", "max_ttl_sum", "max_ttl_count", "average_max_ttl", "processed_at" FROM `traffic_stats` WHERE "src_visibility" = 'all' AND "dst_visibility" = 'all';--> statement-breakpoint +DROP TABLE `traffic_stats`;--> statement-breakpoint +ALTER TABLE `__new_traffic_stats` RENAME TO `traffic_stats`;--> statement-breakpoint +CREATE INDEX `idx_traffic_stats_query` ON `traffic_stats` (`granularity`,`bucket_start`,`source_id`,`ip_version`,`src_locality`,`dst_locality`);--> statement-breakpoint +CREATE INDEX `idx_traffic_stats_timeseries` ON `traffic_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`bucket_start`); ``` ## apps/web/drizzle/0004_dataset_locality.sql Status: added; +1; −0 ```diff @@ -1,0 +1,1 @@ +ALTER TABLE `datasets` ADD `has_locality` integer DEFAULT false NOT NULL; ``` ## apps/web/drizzle/meta/0003_snapshot.json Status: added; +1091; −0 ```diff @@ -1,0 +1,1091 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "fd8995ec-c2ce-4783-b511-a365deea8f51", + "prevId": "0e3d6d54-fced-4c32-bac5-4eb0d8058d5e", + "tables": { + "address_count_stats": { + "name": "address_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_address_count": { + "name": "unique_address_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_count_stats_query": { + "name": "idx_address_count_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "isUnique": false + }, + "idx_address_count_stats_timeseries": { + "name": "idx_address_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "name": "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_count_stats_ip_version_check": { + "name": "address_count_stats_ip_version_check", + "value": "\"address_count_stats\".\"ip_version\" IN (4, 6)" + }, + "address_count_stats_src_locality_check": { + "name": "address_count_stats_src_locality_check", + "value": "\"address_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_count_stats_dst_locality_check": { + "name": "address_count_stats_dst_locality_check", + "value": "\"address_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "address_structure_stats": { + "name": "address_structure_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "structure_kind": { + "name": "structure_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "values_json": { + "name": "values_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_structure_stats_query": { + "name": "idx_address_structure_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "structure_kind" + ], + "isUnique": false + }, + "idx_address_structure_stats_timeseries": { + "name": "idx_address_structure_stats_timeseries", + "columns": [ + "source_id", + "granularity", + "src_locality", + "dst_locality", + "ip_version", + "structure_kind", + "bucket_start" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_structure_kind_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "structure_kind" + ], + "name": "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_structure_kind_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_structure_stats_ip_version_check": { + "name": "address_structure_stats_ip_version_check", + "value": "\"address_structure_stats\".\"ip_version\" IN (4, 6)" + }, + "address_structure_stats_src_locality_check": { + "name": "address_structure_stats_src_locality_check", + "value": "\"address_structure_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_dst_locality_check": { + "name": "address_structure_stats_dst_locality_check", + "value": "\"address_structure_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "bucket_coverage": { + "name": "bucket_coverage", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "coverage_state": { + "name": "coverage_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "observed_units": { + "name": "observed_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expected_units": { + "name": "expected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rejected_units": { + "name": "rejected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_bucket_coverage_query": { + "name": "idx_bucket_coverage_query", + "columns": ["granularity", "bucket_start", "source_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bucket_coverage_source_id_granularity_bucket_start_pk": { + "columns": ["source_id", "granularity", "bucket_start"], + "name": "bucket_coverage_source_id_granularity_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "bucket_coverage_interval_check": { + "name": "bucket_coverage_interval_check", + "value": "\"bucket_coverage\".\"bucket_end\" > \"bucket_coverage\".\"bucket_start\"" + }, + "bucket_coverage_expected_check": { + "name": "bucket_coverage_expected_check", + "value": "\"bucket_coverage\".\"expected_units\" > 0" + }, + "bucket_coverage_observed_check": { + "name": "bucket_coverage_observed_check", + "value": "\"bucket_coverage\".\"observed_units\" >= 0 AND \"bucket_coverage\".\"observed_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_rejected_check": { + "name": "bucket_coverage_rejected_check", + "value": "\"bucket_coverage\".\"rejected_units\" >= 0 AND \"bucket_coverage\".\"rejected_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_state_check": { + "name": "bucket_coverage_state_check", + "value": "(\"bucket_coverage\".\"coverage_state\" = 'complete' AND \"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'unknown' AND \"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'partial' AND NOT (\"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) AND NOT (\"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0))" + } + } + }, + "datasets": { + "name": "datasets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_start_date": { + "name": "default_start_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_mode": { + "name": "source_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "discovery_mode": { + "name": "discovery_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_count_stats": { + "name": "port_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_side": { + "name": "port_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_range": { + "name": "port_range", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_port_count": { + "name": "unique_port_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_port_count_stats_timeseries": { + "name": "idx_port_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "port_side", + "port_range" + ], + "name": "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "port_count_stats_ip_version_check": { + "name": "port_count_stats_ip_version_check", + "value": "\"port_count_stats\".\"ip_version\" IN (4, 6)" + }, + "port_count_stats_src_locality_check": { + "name": "port_count_stats_src_locality_check", + "value": "\"port_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "port_count_stats_dst_locality_check": { + "name": "port_count_stats_dst_locality_check", + "value": "\"port_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "processed_inputs": { + "name": "processed_inputs", + "columns": { + "input_kind": { + "name": "input_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_locator": { + "name": "input_locator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "discovered_at": { + "name": "discovered_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_processed_inputs_source_bucket": { + "name": "idx_processed_inputs_source_bucket", + "columns": ["source_id", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk": { + "columns": ["input_kind", "input_locator", "source_id", "bucket_start"], + "name": "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "protocol_stats": { + "name": "protocol_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_protocols_count": { + "name": "unique_protocols_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocols_list": { + "name": "protocols_list", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_protocol_stats_timeseries": { + "name": "idx_protocol_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "protocol_stats_ip_version_check": { + "name": "protocol_stats_ip_version_check", + "value": "\"protocol_stats\".\"ip_version\" IN (4, 6)" + }, + "protocol_stats_src_locality_check": { + "name": "protocol_stats_src_locality_check", + "value": "\"protocol_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "protocol_stats_dst_locality_check": { + "name": "protocol_stats_dst_locality_check", + "value": "\"protocol_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "source_members": { + "name": "source_members", + "columns": { + "dataset_id": { + "name": "dataset_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "source_members_dataset_id_source_id_member_id_pk": { + "columns": ["dataset_id", "source_id", "member_id"], + "name": "source_members_dataset_id_source_id_member_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "traffic_stats": { + "name": "traffic_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows": { + "name": "flows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_tcp": { + "name": "flows_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_udp": { + "name": "flows_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_icmp": { + "name": "flows_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_other": { + "name": "flows_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets": { + "name": "packets", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_tcp": { + "name": "packets_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_udp": { + "name": "packets_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_icmp": { + "name": "packets_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_other": { + "name": "packets_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_tcp": { + "name": "bytes_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_udp": { + "name": "bytes_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_icmp": { + "name": "bytes_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_other": { + "name": "bytes_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_sum_ms": { + "name": "duration_sum_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_count": { + "name": "duration_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_duration_ms": { + "name": "average_duration_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_ttl_sum": { + "name": "min_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_ttl_count": { + "name": "min_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_min_ttl": { + "name": "average_min_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_ttl_sum": { + "name": "max_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_ttl_count": { + "name": "max_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_max_ttl": { + "name": "average_max_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_traffic_stats_query": { + "name": "idx_traffic_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality" + ], + "isUnique": false + }, + "idx_traffic_stats_timeseries": { + "name": "idx_traffic_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "traffic_stats_ip_version_check": { + "name": "traffic_stats_ip_version_check", + "value": "\"traffic_stats\".\"ip_version\" IN (4, 6)" + }, + "traffic_stats_src_locality_check": { + "name": "traffic_stats_src_locality_check", + "value": "\"traffic_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "traffic_stats_dst_locality_check": { + "name": "traffic_stats_dst_locality_check", + "value": "\"traffic_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": { + "\"address_count_stats\".\"src_visibility\"": "\"address_count_stats\".\"src_locality\"", + "\"address_count_stats\".\"dst_visibility\"": "\"address_count_stats\".\"dst_locality\"", + "\"address_structure_stats\".\"src_visibility\"": "\"address_structure_stats\".\"src_locality\"", + "\"address_structure_stats\".\"dst_visibility\"": "\"address_structure_stats\".\"dst_locality\"", + "\"port_count_stats\".\"src_visibility\"": "\"port_count_stats\".\"src_locality\"", + "\"port_count_stats\".\"dst_visibility\"": "\"port_count_stats\".\"dst_locality\"", + "\"protocol_stats\".\"src_visibility\"": "\"protocol_stats\".\"src_locality\"", + "\"protocol_stats\".\"dst_visibility\"": "\"protocol_stats\".\"dst_locality\"", + "\"traffic_stats\".\"src_visibility\"": "\"traffic_stats\".\"src_locality\"", + "\"traffic_stats\".\"dst_visibility\"": "\"traffic_stats\".\"dst_locality\"" + } + }, + "internal": { + "indexes": {} + } +} ``` ## apps/web/drizzle/meta/0004_snapshot.json Status: added; +1088; −0 ```diff @@ -1,0 +1,1088 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "4a3c1a73-bfe4-4e21-95c9-7ea8a091979f", + "prevId": "fd8995ec-c2ce-4783-b511-a365deea8f51", + "tables": { + "address_count_stats": { + "name": "address_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_address_count": { + "name": "unique_address_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_count_stats_query": { + "name": "idx_address_count_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "isUnique": false + }, + "idx_address_count_stats_timeseries": { + "name": "idx_address_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "name": "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_count_stats_ip_version_check": { + "name": "address_count_stats_ip_version_check", + "value": "\"address_count_stats\".\"ip_version\" IN (4, 6)" + }, + "address_count_stats_src_locality_check": { + "name": "address_count_stats_src_locality_check", + "value": "\"address_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_count_stats_dst_locality_check": { + "name": "address_count_stats_dst_locality_check", + "value": "\"address_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "address_structure_stats": { + "name": "address_structure_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "structure_kind": { + "name": "structure_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "values_json": { + "name": "values_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_structure_stats_query": { + "name": "idx_address_structure_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "structure_kind" + ], + "isUnique": false + }, + "idx_address_structure_stats_timeseries": { + "name": "idx_address_structure_stats_timeseries", + "columns": [ + "source_id", + "granularity", + "src_locality", + "dst_locality", + "ip_version", + "structure_kind", + "bucket_start" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_structure_kind_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "structure_kind" + ], + "name": "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_structure_kind_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_structure_stats_ip_version_check": { + "name": "address_structure_stats_ip_version_check", + "value": "\"address_structure_stats\".\"ip_version\" IN (4, 6)" + }, + "address_structure_stats_src_locality_check": { + "name": "address_structure_stats_src_locality_check", + "value": "\"address_structure_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_dst_locality_check": { + "name": "address_structure_stats_dst_locality_check", + "value": "\"address_structure_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "bucket_coverage": { + "name": "bucket_coverage", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "coverage_state": { + "name": "coverage_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "observed_units": { + "name": "observed_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expected_units": { + "name": "expected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rejected_units": { + "name": "rejected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_bucket_coverage_query": { + "name": "idx_bucket_coverage_query", + "columns": ["granularity", "bucket_start", "source_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bucket_coverage_source_id_granularity_bucket_start_pk": { + "columns": ["source_id", "granularity", "bucket_start"], + "name": "bucket_coverage_source_id_granularity_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "bucket_coverage_interval_check": { + "name": "bucket_coverage_interval_check", + "value": "\"bucket_coverage\".\"bucket_end\" > \"bucket_coverage\".\"bucket_start\"" + }, + "bucket_coverage_expected_check": { + "name": "bucket_coverage_expected_check", + "value": "\"bucket_coverage\".\"expected_units\" > 0" + }, + "bucket_coverage_observed_check": { + "name": "bucket_coverage_observed_check", + "value": "\"bucket_coverage\".\"observed_units\" >= 0 AND \"bucket_coverage\".\"observed_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_rejected_check": { + "name": "bucket_coverage_rejected_check", + "value": "\"bucket_coverage\".\"rejected_units\" >= 0 AND \"bucket_coverage\".\"rejected_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_state_check": { + "name": "bucket_coverage_state_check", + "value": "(\"bucket_coverage\".\"coverage_state\" = 'complete' AND \"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'unknown' AND \"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'partial' AND NOT (\"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) AND NOT (\"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0))" + } + } + }, + "datasets": { + "name": "datasets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_start_date": { + "name": "default_start_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_mode": { + "name": "source_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "discovery_mode": { + "name": "discovery_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_locality": { + "name": "has_locality", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_count_stats": { + "name": "port_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_side": { + "name": "port_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_range": { + "name": "port_range", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_port_count": { + "name": "unique_port_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_port_count_stats_timeseries": { + "name": "idx_port_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "port_side", + "port_range" + ], + "name": "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "port_count_stats_ip_version_check": { + "name": "port_count_stats_ip_version_check", + "value": "\"port_count_stats\".\"ip_version\" IN (4, 6)" + }, + "port_count_stats_src_locality_check": { + "name": "port_count_stats_src_locality_check", + "value": "\"port_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "port_count_stats_dst_locality_check": { + "name": "port_count_stats_dst_locality_check", + "value": "\"port_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "processed_inputs": { + "name": "processed_inputs", + "columns": { + "input_kind": { + "name": "input_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_locator": { + "name": "input_locator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "discovered_at": { + "name": "discovered_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_processed_inputs_source_bucket": { + "name": "idx_processed_inputs_source_bucket", + "columns": ["source_id", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk": { + "columns": ["input_kind", "input_locator", "source_id", "bucket_start"], + "name": "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "protocol_stats": { + "name": "protocol_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_protocols_count": { + "name": "unique_protocols_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocols_list": { + "name": "protocols_list", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_protocol_stats_timeseries": { + "name": "idx_protocol_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "protocol_stats_ip_version_check": { + "name": "protocol_stats_ip_version_check", + "value": "\"protocol_stats\".\"ip_version\" IN (4, 6)" + }, + "protocol_stats_src_locality_check": { + "name": "protocol_stats_src_locality_check", + "value": "\"protocol_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "protocol_stats_dst_locality_check": { + "name": "protocol_stats_dst_locality_check", + "value": "\"protocol_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "source_members": { + "name": "source_members", + "columns": { + "dataset_id": { + "name": "dataset_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "source_members_dataset_id_source_id_member_id_pk": { + "columns": ["dataset_id", "source_id", "member_id"], + "name": "source_members_dataset_id_source_id_member_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "traffic_stats": { + "name": "traffic_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows": { + "name": "flows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_tcp": { + "name": "flows_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_udp": { + "name": "flows_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_icmp": { + "name": "flows_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_other": { + "name": "flows_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets": { + "name": "packets", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_tcp": { + "name": "packets_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_udp": { + "name": "packets_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_icmp": { + "name": "packets_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_other": { + "name": "packets_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_tcp": { + "name": "bytes_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_udp": { + "name": "bytes_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_icmp": { + "name": "bytes_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_other": { + "name": "bytes_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_sum_ms": { + "name": "duration_sum_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_count": { + "name": "duration_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_duration_ms": { + "name": "average_duration_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_ttl_sum": { + "name": "min_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_ttl_count": { + "name": "min_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_min_ttl": { + "name": "average_min_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_ttl_sum": { + "name": "max_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_ttl_count": { + "name": "max_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_max_ttl": { + "name": "average_max_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_traffic_stats_query": { + "name": "idx_traffic_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality" + ], + "isUnique": false + }, + "idx_traffic_stats_timeseries": { + "name": "idx_traffic_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "traffic_stats_ip_version_check": { + "name": "traffic_stats_ip_version_check", + "value": "\"traffic_stats\".\"ip_version\" IN (4, 6)" + }, + "traffic_stats_src_locality_check": { + "name": "traffic_stats_src_locality_check", + "value": "\"traffic_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "traffic_stats_dst_locality_check": { + "name": "traffic_stats_dst_locality_check", + "value": "\"traffic_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} ``` ## apps/web/drizzle/meta/\_journal.json Status: modified; +14; −0 ```diff @@ -21,7 +21,21 @@ "version": "6", "when": 1787518254723, "tag": "0002_keen_moira_mactaggert", + "breakpoints": true + }, + { + "idx": 3, + "version": "6", + "when": 1790239417630, + "tag": "0003_mixed_vin_gonzales", "breakpoints": true + }, + { + "idx": 4, + "version": "6", + "when": 1790313292395, + "tag": "0004_dataset_locality", + "breakpoints": true } ] } ``` ## apps/web/src/lib/components/charts/BreakdownChart.svelte Status: modified; +9; −19 ```diff @@ -11,7 +11,7 @@ import { Skeleton } from '$lib/components/ui/skeleton'; import { navigateToNetflowFile } from '$lib/utils/netflow-file-navigation'; import type { - FlowVisibility, + FlowDirection, IpGranularity, IpMetricKey, ProtocolMetricKey, @@ -97,8 +97,7 @@ availableRouters?: string[]; routers?: RouterConfig; activeMetrics?: MetricsForKind; - srcVisibility?: FlowVisibility; + direction?: FlowDirection; - dstVisibility?: FlowVisibility; onDateChange?: (payload: { startDate: string; endDate: string }) => void; onGroupByChange?: (payload: { groupBy: GroupByOption }) => void; onRouterChange?: (payload: { router: string }) => void; @@ -646,10 +645,7 @@ const labelForSlug = activeLabel ?? label; const slug = generateSlugFromLabel(labelForSlug, '5min'); if (slug) { - void navigateToNetflowFile(goto, slug, props.dataset, { - srcVisibility: props.srcVisibility ?? 'all', - dstVisibility: props.dstVisibility ?? 'all' - }); + void navigateToNetflowFile(goto, slug, props.dataset, props.direction ?? 'all'); } return; } @@ -1181,8 +1177,7 @@ endDate: string; granularity: IpGranularity; routers: string[]; + direction: FlowDirection; - srcVisibility: FlowVisibility; - dstVisibility: FlowVisibility; }; let lastFiltersKey = ''; @@ -1203,8 +1198,7 @@ dataset: props.dataset ?? '', granularity: filters.granularity, routers: filters.routers, - srcVisibility: filters.srcVisibility, - dstVisibility: filters.dstVisibility + direction: filters.direction }); } @@ -1224,8 +1218,7 @@ dataset: props.dataset ?? '', granularity: filters.granularity, routers: filters.routers.join(','), - srcVisibility: filters.srcVisibility, + direction: filters.direction - dstVisibility: filters.dstVisibility }); try { @@ -1338,8 +1331,7 @@ endDate: props.endDate ?? formatDate(today), granularity: props.granularity ?? config.defaultGranularity, routers: selectedRouters, - srcVisibility: props.srcVisibility ?? 'all', - dstVisibility: props.dstVisibility ?? 'all' + direction: props.direction ?? 'all' }; currentGranularity = filters.granularity; @@ -1377,8 +1369,7 @@ const endDateProp = props.endDate; const granularityProp = props.granularity; const nextAddressType = props.addressType ?? 'sa'; - const srcVisibility = props.srcVisibility ?? 'all'; - const dstVisibility = props.dstVisibility ?? 'all'; + const direction = props.direction ?? 'all'; if (nextAddressType !== addressType) { addressType = nextAddressType; @@ -1400,8 +1391,7 @@ // The spectrum card displays one source at a time. Fetching every available source // multiplied its SQL work and response size while the client discarded all but this one. routers: nextRouter ? [nextRouter] : [], - srcVisibility, + direction - dstVisibility }; currentGranularity = filters.granularity; ``` ## apps/web/src/lib/components/charts/flow-characteristics-data.svelte.ts Status: modified; +4; −7 ```diff @@ -5,7 +5,7 @@ import type { GroupByOption, RouterConfig } from '$lib/components/netflow/types'; import type { FlowCharacteristicsResponse, - FlowVisibility, + FlowDirection, IpGranularity, ObservationStats, PortCardinalityCounts, @@ -27,8 +27,7 @@ groupBy: GroupByOption; routers: RouterConfig; routersLoaded: boolean; - srcVisibility: FlowVisibility; - dstVisibility: FlowVisibility; + direction: FlowDirection; }; export type FlowCharacteristicsData = { @@ -64,8 +63,7 @@ dataset: filters.dataset, granularity: GROUP_BY_TO_GRANULARITY[filters.groupBy], routers, + direction: filters.direction - srcVisibility: filters.srcVisibility, - dstVisibility: filters.dstVisibility }); } @@ -163,8 +161,7 @@ dataset: filters.dataset, routers: routers.join(','), granularity, + direction: filters.direction - srcVisibility: filters.srcVisibility, - dstVisibility: filters.dstVisibility }); const controller = new AbortController(); requestController = controller; ``` ## apps/web/src/lib/components/filters/DirectionFilter.svelte Status: added; +28; −0 ```diff @@ -1,0 +1,28 @@ + + + ``` ## apps/web/src/lib/components/filters/PrimaryFilters.svelte Status: modified; +13; −18 ```diff @@ -2,12 +2,13 @@ import { isGranularityAllowedForDateRange } from '$lib/components/charts/chart-utils'; import SegmentedControl from '$lib/components/common/SegmentedControl.svelte'; import DateRangeFilter from '$lib/components/filters/DateRangeFilter.svelte'; + import DirectionFilter from '$lib/components/filters/DirectionFilter.svelte'; import RouterFilter from '$lib/components/filters/RouterFilter.svelte'; import type { GroupByOption, RouterConfig } from '$lib/components/netflow/types.ts'; import { Button } from '$lib/components/ui/button'; import { Card, CardContent, CardHeader, CardTitle } from '$lib/components/ui/card'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { FLOW_SCOPE_OPTIONS, type FlowScopeKey } from '$lib/types/types'; + import type { FlowDirection } from '$lib/types/types'; interface GroupBySelectOption { value: GroupByOption; @@ -27,13 +28,14 @@ endDate: string; groupBy: GroupByOption; routers: RouterConfig; - flowScope: FlowScopeKey; + direction: FlowDirection; + showDirection?: boolean; groupByOptions?: GroupBySelectOption[]; onStartDateChange?: (payload: { startDate: string }) => void; onEndDateChange?: (payload: { endDate: string }) => void; onGroupByChange?: (payload: { groupBy: GroupByOption }) => void; onRoutersChange?: (payload: { routers: RouterConfig }) => void; - onScopeChange?: (payload: { scope: FlowScopeKey }) => void; + onDirectionChange?: (payload: { direction: FlowDirection }) => void; onResetView?: () => void; }>(); @@ -49,9 +51,8 @@ props.onRoutersChange?.({ routers: nextRouters }); } - function handleScopeChange(event: Event) { - const target = event.currentTarget as HTMLSelectElement; - props.onScopeChange?.({ scope: target.value as FlowScopeKey }); + function handleDirectionChange(payload: { direction: FlowDirection }) { + props.onDirectionChange?.(payload); } function handleResetView() { @@ -129,18 +130,12 @@ + {#if props.showDirection ?? true} +
+ Direction: - ``` ## apps/web/src/lib/components/netflow/DatasetDashboardPage.svelte Status: modified; +18; −40 ```diff @@ -16,10 +16,8 @@ import type { Attachment } from 'svelte/attachments'; import { clampGroupByToDateRange } from '$lib/components/charts/chart-utils'; import { - FLOW_SCOPE_OPTIONS, - type FlowVisibility, + type FlowDirection, IP_METRIC_OPTIONS, - type FlowScopeKey, type IpGranularity, type IpMetricKey, type ProtocolMetricKey @@ -32,6 +30,7 @@ const props = $props<{ dataset: string; defaultStartDate: string; + hasLocality?: boolean; routers?: string[]; title?: string; }>(); @@ -142,22 +141,10 @@ }; const ipGranularity = $derived(GROUP_BY_TO_IP[selectedGroupBy]); + const hasLocality = $derived(props.hasLocality ?? true); + const direction = $derived( + hasLocality ? (params.direction as FlowDirection) : 'all' - function getFlowScopeKey( - srcVisibility: FlowVisibility, - dstVisibility: FlowVisibility - ): FlowScopeKey { - return ( - FLOW_SCOPE_OPTIONS.find( - (option) => option.srcVisibility === srcVisibility && option.dstVisibility === dstVisibility - )?.key ?? 'all' - ); - } - - const flowScopeKey = $derived( - getFlowScopeKey(params.srcVisibility as FlowVisibility, params.dstVisibility as FlowVisibility) ); - const srcVisibility = $derived(params.srcVisibility as FlowVisibility); - const dstVisibility = $derived(params.dstVisibility as FlowVisibility); const routers = $derived(Array.isArray(props.routers) ? props.routers : []); const routerStateKey = $derived(`${props.dataset}:${routers.join('\0')}`); const availableSpectrumRouters = $derived(getEnabledRouters(selectedRouters)); @@ -170,8 +157,7 @@ groupBy: selectedGroupBy, routers: selectedRouters, routersLoaded, - srcVisibility, - dstVisibility + direction })); function isValidChartOrder(value: unknown): value is ChartCardId[] { @@ -412,7 +398,7 @@ } function handleMetricNavigateToFile(slug: string) { - void navigateToNetflowFile(goto, slug, props.dataset, { srcVisibility, dstVisibility }); + void navigateToNetflowFile(goto, slug, props.dataset, direction); } function handleRoutersChange(payload: { routers: RouterConfig }) { @@ -432,15 +418,11 @@ ipMetrics = payload.metrics; } - function handleScopeChange(payload: { scope: FlowScopeKey }) { - const flowScope = FLOW_SCOPE_OPTIONS.find((option) => option.key === payload.scope); - if (!flowScope) { + function handleDirectionChange(payload: { direction: FlowDirection }) { + if (payload.direction === params.direction) { return; } - params.update({ - srcVisibility: flowScope.srcVisibility, - dstVisibility: flowScope.dstVisibility + params.direction = payload.direction; - }); } function handleResetView() { @@ -452,8 +434,7 @@ groupBy: selectedGroupBy, startDate, endDate, - srcVisibility: 'all', - dstVisibility: 'all' + direction: 'all' }); } @@ -474,12 +455,13 @@ {endDate} groupBy={selectedGroupBy} routers={selectedRouters} + {direction} + showDirection={hasLocality} - flowScope={flowScopeKey} onStartDateChange={handleStartDateChange} onEndDateChange={handleEndDateChange} onGroupByChange={handleGroupByChange} onRoutersChange={handleRoutersChange} - onScopeChange={handleScopeChange} + onDirectionChange={handleDirectionChange} onResetView={handleResetView} />
@@ -547,8 +529,7 @@ routers={selectedRouters} {routersLoaded} {dataOptions} - {srcVisibility} + {direction} - {dstVisibility} onDateChange={handleDateChange} onGroupByChange={handleGroupByChange} onDataOptionsChange={handleDataOptionsChange} @@ -580,8 +561,7 @@ granularity={ipGranularity} routers={selectedRouters} activeMetrics={ipMetrics} - {srcVisibility} - {dstVisibility} + {direction} onDateChange={handleDateChange} onGroupByChange={handleGroupByChange} onMetricsChange={handleIpMetricsChange} @@ -595,8 +575,7 @@ granularity={ipGranularity} routers={selectedRouters} activeMetrics={protocolMetrics} - {srcVisibility} - {dstVisibility} + {direction} onDateChange={handleDateChange} onGroupByChange={handleGroupByChange} onMetricsChange={(payload) => { @@ -613,8 +592,7 @@ router={selectedSpectrumRouter} addressType={selectedSpectrumAddressType} availableRouters={availableSpectrumRouters} - {srcVisibility} + {direction} - {dstVisibility} onDateChange={handleDateChange} onGroupByChange={handleGroupByChange} onRouterChange={(payload) => { ``` ## apps/web/src/lib/components/netflow/NetflowDashboard.svelte Status: modified; +8; −17 ```diff @@ -21,7 +21,7 @@ RouterConfig } from './types.ts'; import type { - FlowVisibility, + FlowDirection, NetflowIpFamily, NetflowMetricTotals, NetflowStatsResponse, @@ -37,8 +37,7 @@ routers: RouterConfig; routersLoaded: boolean; dataOptions: DataOption[]; - srcVisibility: FlowVisibility; + direction: FlowDirection; - dstVisibility: FlowVisibility; onDateChange?: (payload: { startDate: string; endDate: string }) => void; onGroupByChange?: (payload: { groupBy: GroupByOption }) => void; onDataOptionsChange?: (payload: { options: DataOption[] }) => void; @@ -102,8 +101,7 @@ endDate: string; groupBy: GroupByOption; routers: RouterConfig; + direction: FlowDirection; - srcVisibility: FlowVisibility; - dstVisibility: FlowVisibility; }; let lastFiltersKey = ''; @@ -132,8 +130,7 @@ dataset: props.dataset, groupBy: filters.groupBy, routers: selectedRouters, - srcVisibility: filters.srcVisibility, - dstVisibility: filters.dstVisibility + direction: filters.direction }); } @@ -177,8 +174,7 @@ dataset: props.dataset, routers: selectedRouters.join(','), groupBy: filters.groupBy, + direction: filters.direction - srcVisibility: filters.srcVisibility, - dstVisibility: filters.dstVisibility }); try { @@ -241,10 +237,7 @@ } function handleNavigateToFile(slug: string) { - void navigateToNetflowFile(goto, slug, props.dataset, { - srcVisibility: props.srcVisibility, - dstVisibility: props.dstVisibility + void navigateToNetflowFile(goto, slug, props.dataset, props.direction); - }); } function handleChartTypeChange(newChartType: ChartTypeOption) { @@ -261,8 +254,7 @@ endDate: props.endDate, groupBy: props.groupBy, routers: props.routers, - srcVisibility: props.srcVisibility, + direction: props.direction - dstVisibility: props.dstVisibility }; const selectedRouters = deriveSelectedRouters(filters.routers); @@ -293,8 +285,7 @@ endDate: filters.endDate, groupBy: filters.groupBy, routers: selectedRouters, - srcVisibility: filters.srcVisibility, + direction: filters.direction - dstVisibility: filters.dstVisibility }); if (nextKey === lastFiltersKey) { ``` ## apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte Status: modified; +4; −9 ```diff @@ -5,7 +5,7 @@ import NetflowFileLoadingSkeleton from '$lib/components/netflow/NetflowFileLoadingSkeleton.svelte'; import NetflowFileMessageCard from '$lib/components/netflow/NetflowFileMessageCard.svelte'; import NetflowFileRouterCard from '$lib/components/netflow/NetflowFileRouterCard.svelte'; - import type { FlowVisibility } from '$lib/types/types'; + import type { FlowDirection } from '$lib/types/types'; import { createDateFromPSTComponents, epochToPSTComponents, @@ -16,8 +16,7 @@ type NetflowFileDetailData = { dataset: string; slug: string; - srcVisibility: FlowVisibility; + direction: FlowDirection; - dstVisibility: FlowVisibility; fileInfo: { year: string; month: string; @@ -51,13 +50,9 @@ } const nextSlug = $derived(getNextSlug(data.slug)); - const flowScope = $derived({ - srcVisibility: data.srcVisibility, - dstVisibility: data.dstVisibility - }); function syncLoader() { - loader = getNetflowFileDetailLoader(data.dataset, data.slug, flowScope); + loader = getNetflowFileDetailLoader(data.dataset, data.slug, data.direction); loader.refresh(); } @@ -78,7 +73,7 @@

NetFlow File: {filename}

-
+
``` ## apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts Status: modified; +19; −18 ```diff @@ -1,6 +1,6 @@ import type { FileIpCounts, - FlowScope, + FlowDirection, NetflowFileDetailsResponse, NetflowFileSummaryRecord, SpectrumData, @@ -66,19 +66,16 @@ }; }; +function buildLoaderKey(dataset: string, slug: string, direction: FlowDirection): string { -function buildLoaderKey(dataset: string, slug: string, flowScope: FlowScope): string { - return `${dataset}:${slug}:${flowScope.srcVisibility}:${flowScope.dstVisibility}`; + return `${dataset}:${slug}:${direction}`; } +function buildDirectionSearchParams(direction: FlowDirection): Record { + if (direction === 'all') { -function buildScopeSearchParams(flowScope: FlowScope): Record { - if (flowScope.srcVisibility === 'all' && flowScope.dstVisibility === 'all') { return {}; } - return { + return { direction }; - srcVisibility: flowScope.srcVisibility, - dstVisibility: flowScope.dstVisibility - }; } async function readErrorMessage(response: Response, fallback: string): Promise { @@ -97,7 +94,7 @@ class NetflowFileDetailLoader { readonly dataset: string; readonly slug: string; - readonly flowScope: FlowScope; + readonly direction: FlowDirection; readonly key: string; private _loading = $state(false); @@ -110,11 +107,11 @@ private _lastAccessedAt = Date.now(); private summarySkeletonDelayId: ReturnType | null = null; - constructor(dataset: string, slug: string, flowScope: FlowScope) { + constructor(dataset: string, slug: string, direction: FlowDirection) { this.dataset = dataset; this.slug = slug; + this.direction = direction; + this.key = buildLoaderKey(dataset, slug, direction); - this.flowScope = flowScope; - this.key = buildLoaderKey(dataset, slug, flowScope); } get loading() { @@ -284,7 +281,7 @@ const response = await fetch( `/api/netflow/files/${this.slug}/details?${new URLSearchParams({ dataset: this.dataset, + ...buildDirectionSearchParams(this.direction) - ...buildScopeSearchParams(this.flowScope) }).toString()}` ); @@ -364,7 +361,7 @@ return `/api/netflow/files/${this.slug}/${kind === 'ipCounts' ? 'ip-counts' : kind}?${new URLSearchParams( { dataset: this.dataset, - ...buildScopeSearchParams(this.flowScope), + ...buildDirectionSearchParams(this.direction), router, source: String(side === 'source') } @@ -393,12 +390,16 @@ } } -export function getNetflowFileDetailLoader(dataset: string, slug: string, flowScope: FlowScope) { +export function getNetflowFileDetailLoader( + dataset: string, + slug: string, + direction: FlowDirection +) { - const key = buildLoaderKey(dataset, slug, flowScope); + const key = buildLoaderKey(dataset, slug, direction); let loader = fileDetailLoaders.get(key); if (!loader) { - loader = new NetflowFileDetailLoader(dataset, slug, flowScope); + loader = new NetflowFileDetailLoader(dataset, slug, direction); fileDetailLoaders.set(key, loader); } ``` ## apps/web/src/lib/datasets.ts Status: modified; +1; −0 ```diff @@ -8,6 +8,7 @@ label: z.string().min(1), defaultStartDate: z.iso.date(), discoveryMode: z.string().min(1), + hasLocality: z.boolean(), isDefault: z.boolean() }); ``` ## apps/web/src/lib/schemas.ts Status: modified; +2; −3 ```diff @@ -1,5 +1,5 @@ // schemas.ts -import { FLOW_VISIBILITIES } from '$lib/types/types'; +import { FLOW_DIRECTIONS } from '$lib/types/types'; import { z } from 'zod'; // Requires Zod 4+ @@ -8,7 +8,6 @@ startDate: z.iso.date().default(defaultStartDate), endDate: z.iso.date().default(new Date().toJSON().slice(0, 10)), groupBy: z.enum(['date', 'hour', '30min', '10min', '5min']).default('date'), + direction: z.enum(FLOW_DIRECTIONS).default('all') - srcVisibility: z.enum(FLOW_VISIBILITIES).default('all'), - dstVisibility: z.enum(FLOW_VISIBILITIES).default('all') }); } ``` ## apps/web/src/lib/server/datasets.ts Status: modified; +38; −17 ```diff @@ -10,6 +10,7 @@ defaultStartDate: string; discoveryMode: string; sortOrder: number; + hasLocality: number; }; type LocalDatasetRow = DatasetRow & { @@ -68,6 +69,27 @@ prepare(sql: string): PreparedStatement; } +const localityStatsTables = [ + 'traffic_stats', + 'protocol_stats', + 'address_count_stats', + 'address_structure_stats', + 'port_count_stats' +]; +const currentProductSchemaSql = `SELECT ( + EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'bucket_coverage') + AND EXISTS(SELECT 1 FROM pragma_table_info('datasets') WHERE name = 'has_locality') + ${localityStatsTables + .map( + (table) => `AND ( + NOT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '${table}') + OR (SELECT COUNT(*) FROM pragma_table_info('${table}') + WHERE name IN ('src_locality', 'dst_locality')) = 2 + )` + ) + .join('\n\t')} +) AS compatible`; + const localDbCache = new Map(); const localDbRefreshes = new Map>(); const d1DbCache = new WeakMap(); @@ -362,6 +384,15 @@ return entry.datasetRows.map((row) => ({ ...row })); } + // Backups and obsolete products can remain under data/, but the current dashboard requires + // explicit coverage and locality dimensions and must not let an older database shadow a current + // product with the same ID. + const schema = await entry.db.get<{ compatible: number }>(currentProductSchemaSql); + if (schema?.compatible !== 1) { + entry.datasetRows = []; + return []; + } + const rows = await entry.db.all( ` SELECT @@ -369,25 +400,12 @@ label, default_start_date AS defaultStartDate, discovery_mode AS discoveryMode, + sort_order AS sortOrder, - sort_order AS sortOrder + has_locality AS hasLocality FROM datasets ORDER BY sort_order ASC, id ASC ` ); - // Backups and obsolete products can remain under data/, but the current dashboard requires - // explicit coverage and must not let an older database shadow a current product with the same ID. - const schema = await entry.db.get<{ hasCoverage: number }>( - `SELECT EXISTS( - SELECT 1 - FROM sqlite_master - WHERE type = 'table' AND name = 'bucket_coverage' - ) AS hasCoverage` - ); - if (schema?.hasCoverage !== 1) { - entry.datasetRows = []; - return []; - } - entry.datasetRows = rows.map((row) => ({ ...row, dbPath })); return entry.datasetRows.map((row) => ({ ...row })); } @@ -451,7 +469,8 @@ label, default_start_date AS defaultStartDate, discovery_mode AS discoveryMode, + sort_order AS sortOrder, - sort_order AS sortOrder + has_locality AS hasLocality FROM datasets ORDER BY sort_order ASC, id ASC ` @@ -467,7 +486,8 @@ label, default_start_date AS defaultStartDate, discovery_mode AS discoveryMode, - sort_order AS sortOrder + sort_order AS sortOrder, + has_locality AS hasLocality FROM datasets WHERE id = ? LIMIT 1 @@ -774,6 +794,7 @@ label: dataset.label, defaultStartDate: dataset.defaultStartDate, discoveryMode: dataset.discoveryMode, + hasLocality: dataset.hasLocality === 1, isDefault: dataset.id === defaultDatasetId })); } ``` ## apps/web/src/lib/server/db/local-schema.ts Status: modified; +25; −24 ```diff @@ -5,7 +5,8 @@ default_start_date TEXT NOT NULL, source_mode TEXT DEFAULT 'static' NOT NULL, discovery_mode TEXT DEFAULT 'static' NOT NULL, - sort_order INTEGER DEFAULT 0 NOT NULL + sort_order INTEGER DEFAULT 0 NOT NULL, + has_locality INTEGER DEFAULT 0 NOT NULL ); CREATE TABLE IF NOT EXISTS source_members ( @@ -53,8 +54,8 @@ bucket_start INTEGER NOT NULL, bucket_end INTEGER NOT NULL, ip_version INTEGER NOT NULL CHECK(ip_version IN (4, 6)), + src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), - src_visibility TEXT NOT NULL CHECK(src_visibility IN ('all', 'literal', 'anonymized')), + dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), - dst_visibility TEXT NOT NULL CHECK(dst_visibility IN ('all', 'literal', 'anonymized')), flows INTEGER NOT NULL, flows_tcp INTEGER NOT NULL, flows_udp INTEGER NOT NULL, @@ -80,7 +81,7 @@ max_ttl_count INTEGER NOT NULL, average_max_ttl REAL, processed_at TEXT DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(source_id, granularity, bucket_start, ip_version, src_visibility, dst_visibility) + PRIMARY KEY(source_id, granularity, bucket_start, ip_version, src_locality, dst_locality) ); CREATE TABLE IF NOT EXISTS protocol_stats ( @@ -89,12 +90,12 @@ bucket_start INTEGER NOT NULL, bucket_end INTEGER NOT NULL, ip_version INTEGER NOT NULL CHECK(ip_version IN (4, 6)), - src_visibility TEXT NOT NULL CHECK(src_visibility IN ('all', 'literal', 'anonymized')), + src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), + dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), - dst_visibility TEXT NOT NULL CHECK(dst_visibility IN ('all', 'literal', 'anonymized')), unique_protocols_count INTEGER NOT NULL, protocols_list TEXT NOT NULL, processed_at TEXT DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(source_id, granularity, bucket_start, ip_version, src_visibility, dst_visibility) + PRIMARY KEY(source_id, granularity, bucket_start, ip_version, src_locality, dst_locality) ); CREATE TABLE IF NOT EXISTS address_count_stats ( @@ -103,12 +104,12 @@ bucket_start INTEGER NOT NULL, bucket_end INTEGER NOT NULL, ip_version INTEGER NOT NULL CHECK(ip_version IN (4, 6)), - src_visibility TEXT NOT NULL CHECK(src_visibility IN ('all', 'literal', 'anonymized')), + src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), + dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), - dst_visibility TEXT NOT NULL CHECK(dst_visibility IN ('all', 'literal', 'anonymized')), address_side TEXT NOT NULL CHECK(address_side IN ('source', 'destination')), unique_address_count INTEGER NOT NULL, processed_at TEXT DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(source_id, granularity, bucket_start, ip_version, src_visibility, dst_visibility, address_side) + PRIMARY KEY(source_id, granularity, bucket_start, ip_version, src_locality, dst_locality, address_side) ); CREATE TABLE IF NOT EXISTS address_structure_stats ( @@ -117,8 +118,8 @@ bucket_start INTEGER NOT NULL, bucket_end INTEGER NOT NULL, ip_version INTEGER NOT NULL CHECK(ip_version IN (4, 6)), - src_visibility TEXT NOT NULL CHECK(src_visibility IN ('all', 'literal', 'anonymized')), + src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), + dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), - dst_visibility TEXT NOT NULL CHECK(dst_visibility IN ('all', 'literal', 'anonymized')), address_side TEXT NOT NULL CHECK(address_side IN ('source', 'destination')), structure_kind TEXT NOT NULL CHECK(structure_kind IN ('structure', 'spectrum', 'dimension')), values_json TEXT NOT NULL, @@ -126,7 +127,7 @@ processed_at TEXT DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY( source_id, granularity, bucket_start, ip_version, - src_visibility, dst_visibility, address_side, structure_kind + src_locality, dst_locality, address_side, structure_kind ) ); @@ -136,15 +137,15 @@ bucket_start INTEGER NOT NULL, bucket_end INTEGER NOT NULL, ip_version INTEGER NOT NULL CHECK(ip_version IN (4, 6)), - src_visibility TEXT NOT NULL CHECK(src_visibility IN ('all', 'literal', 'anonymized')), + src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), + dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), - dst_visibility TEXT NOT NULL CHECK(dst_visibility IN ('all', 'literal', 'anonymized')), port_side TEXT NOT NULL CHECK(port_side IN ('source', 'destination')), port_range TEXT NOT NULL CHECK(port_range IN ('low', 'high')), unique_port_count INTEGER NOT NULL, processed_at TEXT DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY( source_id, granularity, bucket_start, ip_version, - src_visibility, dst_visibility, port_side, port_range + src_locality, dst_locality, port_side, port_range ) ); @@ -155,41 +156,41 @@ CREATE INDEX IF NOT EXISTS idx_traffic_stats_query ON traffic_stats ( granularity, bucket_start, source_id, ip_version, - src_visibility, dst_visibility + src_locality, dst_locality ); CREATE INDEX IF NOT EXISTS idx_traffic_stats_timeseries ON traffic_stats ( + source_id, granularity, src_locality, dst_locality, - source_id, granularity, src_visibility, dst_visibility, bucket_start ); CREATE INDEX IF NOT EXISTS idx_protocol_stats_timeseries ON protocol_stats ( - source_id, granularity, src_visibility, dst_visibility, + source_id, granularity, src_locality, dst_locality, bucket_start ); CREATE INDEX IF NOT EXISTS idx_address_count_stats_query ON address_count_stats ( granularity, bucket_start, source_id, ip_version, - src_visibility, dst_visibility, address_side + src_locality, dst_locality, address_side ); CREATE INDEX IF NOT EXISTS idx_address_count_stats_timeseries ON address_count_stats ( + source_id, granularity, src_locality, dst_locality, - source_id, granularity, src_visibility, dst_visibility, bucket_start ); CREATE INDEX IF NOT EXISTS idx_address_structure_stats_query ON address_structure_stats ( granularity, bucket_start, source_id, ip_version, - src_visibility, dst_visibility, address_side, structure_kind + src_locality, dst_locality, address_side, structure_kind ); CREATE INDEX IF NOT EXISTS idx_address_structure_stats_timeseries ON address_structure_stats ( - source_id, granularity, src_visibility, dst_visibility, + source_id, granularity, src_locality, dst_locality, ip_version, structure_kind, bucket_start ); CREATE INDEX IF NOT EXISTS idx_port_count_stats_timeseries ON port_count_stats ( + source_id, granularity, src_locality, dst_locality, - source_id, granularity, src_visibility, dst_visibility, bucket_start ); ``` ## apps/web/src/lib/server/db/schema.ts Status: modified; +83; −42 ```diff @@ -20,7 +20,8 @@ discoveryMode: text('discovery_mode', { enum: ['static', 'live'] }) .notNull() .default('static'), - sortOrder: integer('sort_order').notNull().default(0) + sortOrder: integer('sort_order').notNull().default(0), + hasLocality: integer('has_locality', { mode: 'boolean' }).notNull().default(false) }); export const sourceMembers = sqliteTable( @@ -131,8 +132,8 @@ bucketStart: integer('bucket_start').notNull(), bucketEnd: integer('bucket_end').notNull(), ipVersion: integer('ip_version').notNull(), - srcVisibility: text('src_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), - dstVisibility: text('dst_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), + srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), + dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), ...netflowMetricColumns(), processedAt: text('processed_at').default(currentTimestamp) }, @@ -143,8 +144,8 @@ table.granularity, table.bucketStart, table.ipVersion, - table.srcVisibility, - table.dstVisibility + table.srcLocality, + table.dstLocality ] }), index('idx_traffic_stats_query').on( @@ -152,17 +153,25 @@ table.bucketStart, table.sourceId, table.ipVersion, - table.srcVisibility, - table.dstVisibility + table.srcLocality, + table.dstLocality ), index('idx_traffic_stats_timeseries').on( table.sourceId, table.granularity, - table.srcVisibility, - table.dstVisibility, + table.srcLocality, + table.dstLocality, table.bucketStart ), - check('traffic_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`) + check('traffic_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), + check( + 'traffic_stats_src_locality_check', + sql`${table.srcLocality} IN ('all', 'internal', 'external')` + ), + check( + 'traffic_stats_dst_locality_check', + sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ) ] ); @@ -174,8 +183,8 @@ bucketStart: integer('bucket_start').notNull(), bucketEnd: integer('bucket_end').notNull(), ipVersion: integer('ip_version').notNull(), - srcVisibility: text('src_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), + srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), + dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), - dstVisibility: text('dst_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), uniqueProtocolsCount: integer('unique_protocols_count').notNull(), protocolsList: text('protocols_list').notNull(), processedAt: text('processed_at').default(currentTimestamp) @@ -187,18 +196,26 @@ table.granularity, table.bucketStart, table.ipVersion, - table.srcVisibility, - table.dstVisibility + table.srcLocality, + table.dstLocality ] }), index('idx_protocol_stats_timeseries').on( table.sourceId, table.granularity, - table.srcVisibility, - table.dstVisibility, + table.srcLocality, + table.dstLocality, table.bucketStart ), + check('protocol_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), + check( + 'protocol_stats_src_locality_check', + sql`${table.srcLocality} IN ('all', 'internal', 'external')` + ), + check( + 'protocol_stats_dst_locality_check', - check('protocol_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`) + sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ) ] ); @@ -210,8 +227,8 @@ bucketStart: integer('bucket_start').notNull(), bucketEnd: integer('bucket_end').notNull(), ipVersion: integer('ip_version').notNull(), - srcVisibility: text('src_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), - dstVisibility: text('dst_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), + srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), + dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), addressSide: text('address_side', { enum: ['source', 'destination'] }).notNull(), uniqueAddressCount: integer('unique_address_count').notNull(), processedAt: text('processed_at').default(currentTimestamp) @@ -223,8 +240,8 @@ table.granularity, table.bucketStart, table.ipVersion, + table.srcLocality, - table.srcVisibility, - table.dstVisibility, + table.dstLocality, table.addressSide ] }), @@ -233,18 +250,26 @@ table.bucketStart, table.sourceId, table.ipVersion, + table.srcLocality, - table.srcVisibility, + table.dstLocality, - table.dstVisibility, table.addressSide ), index('idx_address_count_stats_timeseries').on( table.sourceId, table.granularity, - table.srcVisibility, - table.dstVisibility, + table.srcLocality, + table.dstLocality, table.bucketStart ), - check('address_count_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`) + check('address_count_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), + check( + 'address_count_stats_src_locality_check', + sql`${table.srcLocality} IN ('all', 'internal', 'external')` + ), + check( + 'address_count_stats_dst_locality_check', + sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ) ] ); @@ -256,8 +281,8 @@ bucketStart: integer('bucket_start').notNull(), bucketEnd: integer('bucket_end').notNull(), ipVersion: integer('ip_version').notNull(), + srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), + dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), - srcVisibility: text('src_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), - dstVisibility: text('dst_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), portSide: text('port_side', { enum: ['source', 'destination'] }).notNull(), portRange: text('port_range', { enum: ['low', 'high'] }).notNull(), uniquePortCount: integer('unique_port_count').notNull(), @@ -270,8 +295,8 @@ table.granularity, table.bucketStart, table.ipVersion, - table.srcVisibility, - table.dstVisibility, + table.srcLocality, + table.dstLocality, table.portSide, table.portRange ] @@ -279,11 +304,19 @@ index('idx_port_count_stats_timeseries').on( table.sourceId, table.granularity, + table.srcLocality, + table.dstLocality, - table.srcVisibility, - table.dstVisibility, table.bucketStart ), + check('port_count_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), + check( + 'port_count_stats_src_locality_check', + sql`${table.srcLocality} IN ('all', 'internal', 'external')` - check('port_count_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`) + ), + check( + 'port_count_stats_dst_locality_check', + sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ) ] ); @@ -295,8 +328,8 @@ bucketStart: integer('bucket_start').notNull(), bucketEnd: integer('bucket_end').notNull(), ipVersion: integer('ip_version').notNull(), - srcVisibility: text('src_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), - dstVisibility: text('dst_visibility', { enum: ['all', 'literal', 'anonymized'] }).notNull(), + srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), + dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), addressSide: text('address_side', { enum: ['source', 'destination'] }).notNull(), structureKind: text('structure_kind', { enum: ['structure', 'spectrum', 'dimension'] @@ -312,8 +345,8 @@ table.granularity, table.bucketStart, table.ipVersion, - table.srcVisibility, - table.dstVisibility, + table.srcLocality, + table.dstLocality, table.addressSide, table.structureKind ] @@ -323,20 +356,28 @@ table.bucketStart, table.sourceId, table.ipVersion, - table.srcVisibility, + table.srcLocality, + table.dstLocality, - table.dstVisibility, table.addressSide, table.structureKind ), index('idx_address_structure_stats_timeseries').on( table.sourceId, table.granularity, - table.srcVisibility, - table.dstVisibility, + table.srcLocality, + table.dstLocality, table.ipVersion, table.structureKind, table.bucketStart ), + check('address_structure_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`), + check( + 'address_structure_stats_src_locality_check', + sql`${table.srcLocality} IN ('all', 'internal', 'external')` + ), + check( + 'address_structure_stats_dst_locality_check', + sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ) - check('address_structure_stats_ip_version_check', sql`${table.ipVersion} IN (4, 6)`) ] ); ``` ## apps/web/src/lib/server/netflow-v3.ts Status: modified; +31; −39 ```diff @@ -1,8 +1,10 @@ import { + FLOW_DIRECTIONS, + flowDirectionLocalities, - FLOW_VISIBILITIES, IP_GRANULARITIES, + type FlowDirection, + type FlowLocality, - type FlowScope, - type FlowVisibility, + type FlowLocalityPair, type IpGranularity } from '$lib/types/types'; import type { SourceDefinition } from '$lib/server/datasets'; @@ -20,8 +22,9 @@ granularity: IpGranularity; start: number; end: number; - srcVisibility: FlowVisibility; + direction: FlowDirection; + srcLocality: FlowLocality; + dstLocality: FlowLocality; - dstVisibility: FlowVisibility; } export interface RequestValidationError { @@ -34,7 +37,7 @@ export type NetflowSchemaVersion = 'v3'; const VALID_IP_GRANULARITIES = new Set(IP_GRANULARITIES); +const VALID_FLOW_DIRECTIONS = new Set(FLOW_DIRECTIONS); -const VALID_FLOW_VISIBILITIES = new Set(FLOW_VISIBILITIES); export function assertNetflowV3Database(): void { return; @@ -70,52 +73,40 @@ return parseIpGranularity(param) ?? DEFAULT_IP_GRANULARITY; } -export function parseFlowVisibility(param: string | null): FlowVisibility | null { +export function parseFlowDirection(param: string | null): FlowDirection | null { if (!param) { return null; } + return VALID_FLOW_DIRECTIONS.has(param) ? (param as FlowDirection) : null; - return VALID_FLOW_VISIBILITIES.has(param) ? (param as FlowVisibility) : null; } +export function parseDirectionParam(param: string | null): FlowDirection | RequestValidationError { + if (!param) { -export function parseFlowVisibilityParam( + return 'all'; - param: string | null, - name: 'srcVisibility' | 'dstVisibility' -): FlowVisibility | RequestValidationError { - const visibility = parseFlowVisibility(param); - if (visibility) { - return visibility; } - if (!param) { - return 'all'; + const direction = parseFlowDirection(param); + if (direction) { + return direction; } return { - error: `Invalid ${name}. Expected one of: ${FLOW_VISIBILITIES.join(', ')}`, + error: `Invalid direction. Expected one of: ${FLOW_DIRECTIONS.join(', ')}`, status: 400 }; } -export function parseFlowScopeParams(url: URL): FlowScope | RequestValidationError { - const srcVisibility = parseFlowVisibilityParam( - url.searchParams.get('srcVisibility'), - 'srcVisibility' - ); - const dstVisibility = parseFlowVisibilityParam( - url.searchParams.get('dstVisibility'), - 'dstVisibility' +export function parseFlowDirectionParams( + url: URL +): ({ direction: FlowDirection } & FlowLocalityPair) | RequestValidationError { - ); + const direction = parseDirectionParam(url.searchParams.get('direction')); - if (typeof srcVisibility !== 'string') { - return srcVisibility; + if (typeof direction !== 'string') { + return direction; } - if (typeof dstVisibility !== 'string') { + return { direction, ...flowDirectionLocalities(direction) }; - return dstVisibility; - } - - return { srcVisibility, dstVisibility }; } export function parseAggregateStatsParams(url: URL): AggregateStatsParams | RequestValidationError { @@ -125,7 +116,7 @@ const granularity = parsedGranularity ?? DEFAULT_IP_GRANULARITY; const start = parseTimestamp(url.searchParams.get('startDate')); const end = parseTimestamp(url.searchParams.get('endDate')); - const flowScope = parseFlowScopeParams(url); + const flowDirection = parseFlowDirectionParams(url); if (routers.length === 0) { return { error: 'No routers selected', status: 400 }; @@ -138,8 +129,8 @@ }; } + if ('error' in flowDirection) { - if ('error' in flowScope) { - return flowScope; + return flowDirection; } if (start === null || end === null) { @@ -155,8 +146,9 @@ granularity, start, end, - srcVisibility: flowScope.srcVisibility, - dstVisibility: flowScope.dstVisibility + direction: flowDirection.direction, + srcLocality: flowDirection.srcLocality, + dstLocality: flowDirection.dstLocality }; } ``` [Next page](https://git.cafe/flambo/atlantis/pull/108/diffs?after=ImFwcHMvd2ViL3NyYy9saWIvc2VydmVyL25ldGZsb3ctdjMudHMi&expectedVersion=0&format=markdown)