{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://schemas.fermihdi.io/hd/dataset_schema.json", "title": "FermiHDI HD Dataset Schema", "description": "The JSON Schema description for a dataset held in a FermiHDI HD system", "type": "object", "properties": { "version": { "description": "The version of the dataset schema", "type": "integer" }, "datasets": { "description": "An Array of dataset descriptions", "type": "array", "items": { "type": "object", "properties": { "id": { "description": "A UID unique identifier for the dataset", "type": "string" }, "name": { "description": "The name of the dataset", "type": "string" }, "purpose": { "description": "The purpose of the field", "type": "string" }, "record_len": { "description": "Records are stored as a concatenated byte array, the fields are the number of bytes in a record", "type": "integer" }, "dataset_type": { "description": "The type of the dataset `flow` describes network-flow records and was in use before it was declared here.", "type": "string", "enum": [ "time_series", "flow", "other" ] }, "batch_id_field": { "description": "Optional - Defines a field used for hashing (batch_id) that should be used on ingest; Note: unless this field is spcifed in the fields section it will not be included in resulting record.", "type": "object", "properties": { "name": { "description": "The name of the field in the dataset that should be used as the batch_id; Note: unless also included belowthis field will not be included in the record", "type": "string" }, "round_batch_id_field": { "description": "If true, rounds the extracted batch ID to the time index interval.", "type": "boolean" }, "field_type": { "description": "Var type The sized integer names are accepted alongside the generic `integer`, because deployed dataset configurations already use them.", "type": "string", "enum": [ "string", "integer", "float", "double", "boolean", "uint8", "int8", "uint16", "int16", "uint32", "int32", "uint64", "int64", "uint128", "int128", "uint256", "int256" ] } }, "required": [ "name", "field_type" ] }, "time_index_field": { "description": "The name of the field in the dataset that is a timestamp used for indexing the records. Names a field in `fields`; it is not an index or an offset.", "type": "string" }, "time_index_interval": { "description": "The number of time_index_interval_unit increments per batch bucket. For example: interval=60 with unit='seconds' creates 60-second batches; interval=60000 with unit='milliseconds' also creates 60-second batches. When batch_id_field is used in a time_series dataset, the field value is expected to be in time_index_interval_unit units.", "type": "integer" }, "time_index_interval_unit": { "description": "The unit for both the time_index_interval count and the epoch value encoded in the batch_id. Defaults to 'seconds' for backward compatibility. Example: unit='milliseconds' with interval=60000 creates 60-second batches, with the batch_id epoch stored as a millisecond-precision Unix timestamp.", "type": "string", "enum": [ "seconds", "milliseconds", "nanoseconds", "minutes" ], "default": "seconds" }, "compressed_time_fields": { "description": "An Array of time fields in the dataset which are time compressed; Note: time compression is based on the use of a unix timestamp at second level accuracy and that the time fields are deltas from that batch_id.", "type": "array", "items": { "type": "object", "properties": { "name": { "description": "Time field name", "type": "string" }, "time_compression": { "description": "If there is time compression, what is the level - This assumes that the batch_id id a Unix Timestamp at second level accuracy and that the time fields are deltas from that batch_id.", "type": "string", "enum": [ "none", "minutes", "seconds", "milliseconds", "nanoseconds" ] } }, "required": [ "name" ] } }, "storage_query_targets": { "description": "An Array of HD storage cluster controller IP addresses or gateways May be empty when targets are discovered at runtime via the DMS or the proxy rather than pinned in the dataset.", "type": "array", "items": { "type": "object", "properties": { "address": { "description": "The IP address or gateway of the HD storage cluster controller", "type": "string" }, "port": { "description": "The port number of the HD storage cluster controller", "type": "integer" }, "description": { "description": "Description of the storage cluster controller", "type": "string" } }, "required": [ "address", "port" ] } }, "fields": { "type": "array", "items": { "type": "object", "properties": { "name": { "description": "Field name", "type": "string" }, "type": { "description": "Stored type of the field, occupying exactly the bytes from first_byte_position to last_byte_position.\n\nInteger widths above 64 bits are carried, but readers currently interpret at most 64 bits of value; the remaining high bytes are expected to be zero.\n\n`bytes`, `ipv4`, `ipv6` and `timestamp` were already implemented by the SDK and the ingesters before they were declared here.\n\n`blob` is a reference, not a payload. The field holds a 64-byte UTF-8 object key and nothing else; the object itself lives in an S3-compatible bucket. The full key is composed at call time as `/` \u2014 the batch prefix is never stored, so a dataset declaring a blob field must also declare batch_id_field, or a reader cannot rebuild the prefix.\n\nThe ingester writes the object and the SDK retrieves it; the SDK does not write blobs. Grouping objects under the batch prefix means discarding a bad batch is a prefix delete rather than a per-object sweep.\n\nKeys are generated, not caller-supplied: 64 bytes holds a SHA-256 hex digest or a UUID, and leaves no room for a path.", "type": "string", "enum": [ "uint8", "int8", "uint16", "int16", "uint32", "int32", "uint64", "int64", "uint128", "int128", "uint256", "int256", "float", "double", "boolean", "numeric", "string", "bytes", "blob", "ipv4", "ipv6", "timestamp" ] }, "purpose": { "description": "The purpose of the field", "type": "string" }, "first_byte_position": { "description": "The first index (inclusive) in a record bytearray used by this field's data", "type": "integer" }, "last_byte_position": { "description": "The last index (exclusive - Up to but not including: eg. a ending index of 4 and starting index of 0 would result in the indexes 0, 1, 2, and 3 being used) in a record bytearray used by this field's data", "type": "integer" }, "numeric_precision": { "description": "Total number of significant decimal digits. Required when type is `numeric`, ignored otherwise. Must fit the field width: a decimal128 representation occupies 16 bytes and supports a precision up to 38.", "type": "integer", "minimum": 1, "maximum": 76 }, "numeric_scale": { "description": "Number of digits to the right of the decimal point. Required when type is `numeric`, ignored otherwise. Must not exceed numeric_precision.", "type": "integer", "minimum": 0 }, "string_encoding": { "description": "Encoding of the SOURCE data for this field. Applies to `string` fields only.\n\nRecords are always stored UTF-32 in the Storage Cluster: the ingester and the SDK convert on the way in, so the cluster holds exactly one encoding. In UTF-32 every code point occupies four bytes, so a byte-offset filter is always code-point aligned and a match can never straddle a character \u2014 which is what makes byte-pattern filtering correct rather than lucky.\n\nTwo consequences. Field widths are budgeted in UTF-32, so a 64-byte string field holds 16 code points, not 64 characters. And ASCII-heavy data occupies four times the bytes at rest; that is the price of alignment-free filtering.\n\nDoes NOT apply to `blob`, whose object key is always UTF-8 because S3 defines object keys that way.", "type": "string", "enum": [ "ascii", "utf-8", "utf-16", "utf-32" ], "default": "utf-8" }, "blob_store": { "description": "Optional name of the configured blob store this field's objects live in, when a deployment defines more than one. Omitted, the component's default blob_store configuration is used.", "type": "string" } }, "required": [ "name", "type", "first_byte_position", "last_byte_position" ], "allOf": [ { "if": { "properties": { "type": { "const": "numeric" } }, "required": [ "type" ] }, "then": { "required": [ "numeric_precision", "numeric_scale" ] } } ] }, "minItems": 1 } } }, "minItems": 1 } }, "required": [ "datasets" ] }