diff --git a/CMakeLists.txt b/CMakeLists.txt index 3bfe9f7..3a1aaec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,6 +82,7 @@ set(AMDUAT_ENC_SRCS src/near_core/enc/fer1_receipt.c src/near_core/fer/receipt.c src/near_core/enc/asl_log.c + src/near_core/enc/asl_core_index.c src/near_core/enc/pel_program_dag.c src/near_core/enc/pel_program_dag_desc.c src/near_core/enc/pel_trace_dag.c @@ -258,6 +259,16 @@ target_link_libraries(amduat_test_asl_log ) add_test(NAME asl_log COMMAND amduat_test_asl_log) +add_executable(amduat_test_asl_core_index tests/enc/test_asl_core_index.c) +target_include_directories(amduat_test_asl_core_index + PRIVATE ${AMDUAT_INTERNAL_DIR} + PRIVATE ${AMDUAT_INCLUDE_DIR} +) +target_link_libraries(amduat_test_asl_core_index + PRIVATE amduat_enc amduat_hash_asl1 amduat_asl amduat_util +) +add_test(NAME asl_core_index COMMAND amduat_test_asl_core_index) + add_executable(amduat_test_tgk1_edge tests/enc/test_tgk1_edge.c) target_include_directories(amduat_test_tgk1_edge PRIVATE ${AMDUAT_INTERNAL_DIR} diff --git a/include/amduat/enc/asl_core_index.h b/include/amduat/enc/asl_core_index.h new file mode 100644 index 0000000..36d9c96 --- /dev/null +++ b/include/amduat/enc/asl_core_index.h @@ -0,0 +1,101 @@ +#ifndef AMDUAT_ENC_ASL_CORE_INDEX_H +#define AMDUAT_ENC_ASL_CORE_INDEX_H + +#include "amduat/asl/core.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +enum { + AMDUAT_ASL_CORE_INDEX_VERSION = 3, + AMDUAT_ASL_CORE_INDEX_HEADER_SIZE = 112, + AMDUAT_ASL_CORE_INDEX_RECORD_SIZE = 48, + AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE = 16, + AMDUAT_ASL_CORE_INDEX_FOOTER_SIZE = 24 +}; + +enum { + AMDUAT_ASL_INDEX_FLAG_TOMBSTONE = 0x00000001u +}; + +typedef struct { + uint64_t magic; + uint16_t version; + uint16_t shard_id; + uint32_t header_size; + uint64_t snapshot_min; + uint64_t snapshot_max; + uint64_t record_count; + uint64_t records_offset; + uint64_t bloom_offset; + uint64_t bloom_size; + uint64_t digests_offset; + uint64_t digests_size; + uint64_t extents_offset; + uint64_t extent_count; + uint32_t segment_domain_id; + uint8_t segment_visibility; + uint8_t federation_version; + uint16_t reserved0; + uint64_t flags; +} amduat_asl_segment_header_t; + +typedef struct { + uint32_t hash_id; + uint16_t digest_len; + uint16_t reserved0; + uint64_t digest_offset; + uint64_t extents_offset; + uint32_t extent_count; + uint32_t total_length; + uint32_t domain_id; + uint8_t visibility; + uint8_t has_cross_domain_source; + uint16_t reserved1; + uint32_t cross_domain_source; + uint32_t flags; +} amduat_asl_index_record_t; + +typedef struct { + uint64_t block_id; + uint32_t offset; + uint32_t length; +} amduat_asl_extent_record_t; + +typedef struct { + uint64_t crc64; + uint64_t seal_snapshot; + uint64_t seal_time_ns; +} amduat_asl_segment_footer_t; + +typedef struct { + amduat_asl_segment_header_t header; + amduat_octets_t bloom; + amduat_asl_index_record_t *records; + size_t record_count; + amduat_octets_t digests; + amduat_asl_extent_record_t *extents; + size_t extent_count; + amduat_asl_segment_footer_t footer; +} amduat_asl_core_index_segment_t; + +bool amduat_enc_asl_core_index_encode_v1( + const amduat_asl_core_index_segment_t *segment, + amduat_octets_t *out_bytes); + +bool amduat_enc_asl_core_index_decode_v1( + amduat_octets_t bytes, + amduat_asl_core_index_segment_t *out_segment); + +void amduat_enc_asl_core_index_free(amduat_asl_core_index_segment_t *segment); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* AMDUAT_ENC_ASL_CORE_INDEX_H */ diff --git a/src/near_core/enc/asl_core_index.c b/src/near_core/enc/asl_core_index.c new file mode 100644 index 0000000..5fe06ce --- /dev/null +++ b/src/near_core/enc/asl_core_index.c @@ -0,0 +1,812 @@ +#include "amduat/enc/asl_core_index.h" + +#include +#include +#include + +enum { + AMDUAT_ASL_CORE_INDEX_MAGIC = 0x33305844494c5341ull +}; + +static void amduat_asl_core_index_store_u16_le(uint8_t *out, + uint16_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); +} + +static void amduat_asl_core_index_store_u32_le(uint8_t *out, + uint32_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); +} + +static void amduat_asl_core_index_store_u64_le(uint8_t *out, + uint64_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); + out[4] = (uint8_t)((value >> 32) & 0xffu); + out[5] = (uint8_t)((value >> 40) & 0xffu); + out[6] = (uint8_t)((value >> 48) & 0xffu); + out[7] = (uint8_t)((value >> 56) & 0xffu); +} + +static uint16_t amduat_asl_core_index_load_u16_le(const uint8_t *data) { + return (uint16_t)data[0] | ((uint16_t)data[1] << 8); +} + +static uint32_t amduat_asl_core_index_load_u32_le(const uint8_t *data) { + return (uint32_t)data[0] | ((uint32_t)data[1] << 8) | + ((uint32_t)data[2] << 16) | ((uint32_t)data[3] << 24); +} + +static uint64_t amduat_asl_core_index_load_u64_le(const uint8_t *data) { + return (uint64_t)data[0] | ((uint64_t)data[1] << 8) | + ((uint64_t)data[2] << 16) | ((uint64_t)data[3] << 24) | + ((uint64_t)data[4] << 32) | ((uint64_t)data[5] << 40) | + ((uint64_t)data[6] << 48) | ((uint64_t)data[7] << 56); +} + +static bool amduat_asl_core_index_add_size(size_t *acc, size_t add) { + if (*acc > SIZE_MAX - add) { + return false; + } + *acc += add; + return true; +} + +static bool amduat_asl_core_index_is_aligned8(uint64_t value) { + return (value & 7u) == 0u; +} + +static uint64_t amduat_asl_core_index_crc64(const uint8_t *data, size_t len) { + uint64_t crc = 0u; + size_t i; + + for (i = 0; i < len; ++i) { + uint64_t bit; + uint8_t value = data[i]; + crc ^= ((uint64_t)value) << 56; + for (bit = 0; bit < 8; ++bit) { + if (crc & 0x8000000000000000ull) { + crc = (crc << 1) ^ 0x42f0e1eba9ea3693ull; + } else { + crc <<= 1; + } + } + } + return crc; +} + +static bool amduat_asl_core_index_validate_record( + const amduat_asl_index_record_t *record, + const amduat_asl_extent_record_t *extents, + size_t extent_count, + size_t *extent_cursor, + size_t *digest_cursor, + size_t digests_len, + uint8_t *max_visibility) { + size_t i; + size_t start; + uint64_t total_len; + bool is_tombstone; + + if (record->reserved0 != 0 || record->reserved1 != 0) { + return false; + } + if ((record->flags & ~AMDUAT_ASL_INDEX_FLAG_TOMBSTONE) != 0) { + return false; + } + if (record->visibility > 1) { + return false; + } + if (record->has_cross_domain_source > 1) { + return false; + } + if (record->has_cross_domain_source == 0 && + record->cross_domain_source != 0) { + return false; + } + if (record->digest_len == 0) { + return false; + } + if ((size_t)record->digest_len > digests_len - *digest_cursor) { + return false; + } + *digest_cursor += record->digest_len; + + is_tombstone = (record->flags & AMDUAT_ASL_INDEX_FLAG_TOMBSTONE) != 0; + if (is_tombstone) { + if (record->extent_count != 0 || record->total_length != 0) { + return false; + } + if (record->extents_offset != 0) { + return false; + } + if (record->visibility > *max_visibility) { + *max_visibility = record->visibility; + } + return true; + } + + if (record->extent_count == 0) { + return false; + } + if ((size_t)record->extent_count > extent_count - *extent_cursor) { + return false; + } + + start = *extent_cursor; + total_len = 0; + for (i = 0; i < record->extent_count; ++i) { + const amduat_asl_extent_record_t *extent = &extents[start + i]; + if (extent->length == 0) { + return false; + } + total_len += extent->length; + if (total_len > UINT32_MAX) { + return false; + } + } + if ((uint32_t)total_len != record->total_length) { + return false; + } + *extent_cursor += record->extent_count; + + if (record->visibility > *max_visibility) { + *max_visibility = record->visibility; + } + + return true; +} + +bool amduat_enc_asl_core_index_encode_v1( + const amduat_asl_core_index_segment_t *segment, + amduat_octets_t *out_bytes) { + size_t i; + size_t offset; + size_t digest_cursor; + size_t extent_cursor; + size_t record_count; + size_t extent_count; + size_t total_len; + uint64_t header_size; + uint64_t bloom_offset; + uint64_t bloom_size; + uint64_t records_offset; + uint64_t records_bytes; + uint64_t digests_offset; + uint64_t digests_size; + uint64_t extents_offset; + uint64_t extents_bytes; + uint64_t footer_offset; + uint8_t *buffer; + uint8_t max_visibility; + + if (out_bytes == NULL) { + return false; + } + out_bytes->data = NULL; + out_bytes->len = 0; + + if (segment == NULL) { + return false; + } + + record_count = segment->record_count; + extent_count = segment->extent_count; + + if (record_count != 0 && segment->records == NULL) { + return false; + } + if (extent_count != 0 && segment->extents == NULL) { + return false; + } + if (segment->digests.len != 0 && segment->digests.data == NULL) { + return false; + } + if (segment->bloom.len != 0 && segment->bloom.data == NULL) { + return false; + } + if (segment->header.flags != 0 || segment->header.federation_version != 0 || + segment->header.reserved0 != 0) { + return false; + } + if (segment->header.segment_visibility > 1) { + return false; + } + + digest_cursor = 0; + extent_cursor = 0; + max_visibility = 0; + for (i = 0; i < record_count; ++i) { + if (!amduat_asl_core_index_validate_record( + &segment->records[i], + segment->extents, + extent_count, + &extent_cursor, + &digest_cursor, + segment->digests.len, + &max_visibility)) { + return false; + } + } + if (digest_cursor != segment->digests.len) { + return false; + } + if (extent_cursor != extent_count) { + return false; + } + if (segment->header.segment_visibility != max_visibility) { + return false; + } + + if (segment->bloom.len != 0 && (segment->bloom.len % 8u) != 0u) { + return false; + } + if ((segment->digests.len % 8u) != 0u) { + return false; + } + + if (record_count > UINT64_MAX / AMDUAT_ASL_CORE_INDEX_RECORD_SIZE) { + return false; + } + if (extent_count > UINT64_MAX / AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE) { + return false; + } + + header_size = AMDUAT_ASL_CORE_INDEX_HEADER_SIZE; + bloom_size = segment->bloom.len; + if (bloom_size == 0) { + bloom_offset = 0; + } else { + bloom_offset = header_size; + } + records_offset = header_size + bloom_size; + if (!amduat_asl_core_index_is_aligned8(records_offset)) { + return false; + } + + records_bytes = (uint64_t)record_count * + AMDUAT_ASL_CORE_INDEX_RECORD_SIZE; + digests_offset = records_offset + records_bytes; + if (digests_offset < records_offset || + !amduat_asl_core_index_is_aligned8(digests_offset)) { + return false; + } + digests_size = segment->digests.len; + extents_offset = digests_offset + digests_size; + if (extents_offset < digests_offset || + !amduat_asl_core_index_is_aligned8(extents_offset)) { + return false; + } + extents_bytes = (uint64_t)extent_count * + AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE; + footer_offset = extents_offset + extents_bytes; + if (footer_offset < extents_offset) { + return false; + } + if (footer_offset > SIZE_MAX) { + return false; + } + + total_len = 0; + if (!amduat_asl_core_index_add_size(&total_len, (size_t)footer_offset) || + !amduat_asl_core_index_add_size(&total_len, + AMDUAT_ASL_CORE_INDEX_FOOTER_SIZE)) { + return false; + } + + buffer = (uint8_t *)malloc(total_len); + if (buffer == NULL) { + return false; + } + + offset = 0; + amduat_asl_core_index_store_u64_le(buffer + offset, + AMDUAT_ASL_CORE_INDEX_MAGIC); + offset += 8; + amduat_asl_core_index_store_u16_le(buffer + offset, + AMDUAT_ASL_CORE_INDEX_VERSION); + offset += 2; + amduat_asl_core_index_store_u16_le(buffer + offset, + segment->header.shard_id); + offset += 2; + amduat_asl_core_index_store_u32_le(buffer + offset, + AMDUAT_ASL_CORE_INDEX_HEADER_SIZE); + offset += 4; + amduat_asl_core_index_store_u64_le(buffer + offset, + segment->header.snapshot_min); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, + segment->header.snapshot_max); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, record_count); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, records_offset); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, bloom_offset); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, bloom_size); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, digests_offset); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, digests_size); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, extents_offset); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, extent_count); + offset += 8; + amduat_asl_core_index_store_u32_le(buffer + offset, + segment->header.segment_domain_id); + offset += 4; + buffer[offset++] = segment->header.segment_visibility; + buffer[offset++] = 0; + amduat_asl_core_index_store_u16_le(buffer + offset, 0); + offset += 2; + amduat_asl_core_index_store_u64_le(buffer + offset, 0); + offset += 8; + + if (offset != AMDUAT_ASL_CORE_INDEX_HEADER_SIZE) { + free(buffer); + return false; + } + + if (bloom_size != 0) { + memcpy(buffer + (size_t)bloom_offset, segment->bloom.data, + (size_t)bloom_size); + } + + digest_cursor = 0; + extent_cursor = 0; + offset = (size_t)records_offset; + for (i = 0; i < record_count; ++i) { + const amduat_asl_index_record_t *record = &segment->records[i]; + uint64_t digest_offset = digests_offset + digest_cursor; + uint64_t extents_offset_out = 0; + bool is_tombstone = + (record->flags & AMDUAT_ASL_INDEX_FLAG_TOMBSTONE) != 0; + + if (!is_tombstone) { + extents_offset_out = extents_offset + + (uint64_t)extent_cursor * + AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE; + extent_cursor += record->extent_count; + } + + amduat_asl_core_index_store_u32_le(buffer + offset, record->hash_id); + offset += 4; + amduat_asl_core_index_store_u16_le(buffer + offset, record->digest_len); + offset += 2; + amduat_asl_core_index_store_u16_le(buffer + offset, 0); + offset += 2; + amduat_asl_core_index_store_u64_le(buffer + offset, digest_offset); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, extents_offset_out); + offset += 8; + amduat_asl_core_index_store_u32_le(buffer + offset, + record->extent_count); + offset += 4; + amduat_asl_core_index_store_u32_le(buffer + offset, + record->total_length); + offset += 4; + amduat_asl_core_index_store_u32_le(buffer + offset, record->domain_id); + offset += 4; + buffer[offset++] = record->visibility; + buffer[offset++] = record->has_cross_domain_source; + amduat_asl_core_index_store_u16_le(buffer + offset, 0); + offset += 2; + amduat_asl_core_index_store_u32_le(buffer + offset, + record->cross_domain_source); + offset += 4; + amduat_asl_core_index_store_u32_le(buffer + offset, record->flags); + offset += 4; + + digest_cursor += record->digest_len; + } + + if (segment->digests.len != 0) { + memcpy(buffer + (size_t)digests_offset, segment->digests.data, + segment->digests.len); + } + + offset = (size_t)extents_offset; + for (i = 0; i < extent_count; ++i) { + const amduat_asl_extent_record_t *extent = &segment->extents[i]; + amduat_asl_core_index_store_u64_le(buffer + offset, extent->block_id); + offset += 8; + amduat_asl_core_index_store_u32_le(buffer + offset, extent->offset); + offset += 4; + amduat_asl_core_index_store_u32_le(buffer + offset, extent->length); + offset += 4; + } + + { + uint64_t crc = + amduat_asl_core_index_crc64(buffer, (size_t)footer_offset); + offset = (size_t)footer_offset; + amduat_asl_core_index_store_u64_le(buffer + offset, crc); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, + segment->footer.seal_snapshot); + offset += 8; + amduat_asl_core_index_store_u64_le(buffer + offset, + segment->footer.seal_time_ns); + offset += 8; + } + + out_bytes->data = buffer; + out_bytes->len = total_len; + return true; +} + +bool amduat_enc_asl_core_index_decode_v1( + amduat_octets_t bytes, + amduat_asl_core_index_segment_t *out_segment) { + const uint8_t *data; + amduat_asl_segment_header_t header; + amduat_asl_segment_footer_t footer; + size_t record_count; + size_t extent_count; + size_t i; + uint64_t footer_offset; + uint64_t records_bytes; + uint64_t extents_bytes; + uint8_t max_visibility; + bool legacy_defaults; + + if (out_segment == NULL) { + return false; + } + memset(out_segment, 0, sizeof(*out_segment)); + + if (bytes.len != 0 && bytes.data == NULL) { + return false; + } + if (bytes.len < AMDUAT_ASL_CORE_INDEX_HEADER_SIZE + + AMDUAT_ASL_CORE_INDEX_FOOTER_SIZE) { + return false; + } + + data = bytes.data; + header.magic = amduat_asl_core_index_load_u64_le(data); + header.version = amduat_asl_core_index_load_u16_le(data + 8); + header.shard_id = amduat_asl_core_index_load_u16_le(data + 10); + header.header_size = amduat_asl_core_index_load_u32_le(data + 12); + header.snapshot_min = amduat_asl_core_index_load_u64_le(data + 16); + header.snapshot_max = amduat_asl_core_index_load_u64_le(data + 24); + header.record_count = amduat_asl_core_index_load_u64_le(data + 32); + header.records_offset = amduat_asl_core_index_load_u64_le(data + 40); + header.bloom_offset = amduat_asl_core_index_load_u64_le(data + 48); + header.bloom_size = amduat_asl_core_index_load_u64_le(data + 56); + header.digests_offset = amduat_asl_core_index_load_u64_le(data + 64); + header.digests_size = amduat_asl_core_index_load_u64_le(data + 72); + header.extents_offset = amduat_asl_core_index_load_u64_le(data + 80); + header.extent_count = amduat_asl_core_index_load_u64_le(data + 88); + header.segment_domain_id = amduat_asl_core_index_load_u32_le(data + 96); + header.segment_visibility = data[100]; + header.federation_version = data[101]; + header.reserved0 = amduat_asl_core_index_load_u16_le(data + 102); + header.flags = amduat_asl_core_index_load_u64_le(data + 104); + + if (header.magic != AMDUAT_ASL_CORE_INDEX_MAGIC) { + return false; + } + if (header.version < 1 || header.version > AMDUAT_ASL_CORE_INDEX_VERSION) { + return false; + } + if (header.header_size != AMDUAT_ASL_CORE_INDEX_HEADER_SIZE) { + return false; + } + if (header.flags != 0) { + return false; + } + + legacy_defaults = header.version < AMDUAT_ASL_CORE_INDEX_VERSION; + if (!legacy_defaults) { + if (header.federation_version != 0 || header.reserved0 != 0) { + return false; + } + if (header.segment_visibility > 1) { + return false; + } + } + + if (header.record_count > SIZE_MAX) { + return false; + } + record_count = (size_t)header.record_count; + if (record_count > UINT64_MAX / AMDUAT_ASL_CORE_INDEX_RECORD_SIZE) { + return false; + } + records_bytes = header.record_count * + AMDUAT_ASL_CORE_INDEX_RECORD_SIZE; + + if (header.extent_count > SIZE_MAX) { + return false; + } + extent_count = (size_t)header.extent_count; + if (extent_count > UINT64_MAX / AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE) { + return false; + } + extents_bytes = header.extent_count * + AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE; + + if (header.bloom_size == 0) { + if (header.bloom_offset != 0) { + return false; + } + } else { + if (header.bloom_offset != AMDUAT_ASL_CORE_INDEX_HEADER_SIZE) { + return false; + } + if (!amduat_asl_core_index_is_aligned8(header.bloom_size)) { + return false; + } + } + + if (header.records_offset != AMDUAT_ASL_CORE_INDEX_HEADER_SIZE + + header.bloom_size) { + return false; + } + if (!amduat_asl_core_index_is_aligned8(header.records_offset)) { + return false; + } + if (header.digests_offset != header.records_offset + records_bytes) { + return false; + } + if (!amduat_asl_core_index_is_aligned8(header.digests_offset)) { + return false; + } + if (!amduat_asl_core_index_is_aligned8(header.digests_size)) { + return false; + } + if (header.extents_offset != header.digests_offset + header.digests_size) { + return false; + } + if (!amduat_asl_core_index_is_aligned8(header.extents_offset)) { + return false; + } + + footer_offset = header.extents_offset + extents_bytes; + if (footer_offset < header.extents_offset) { + return false; + } + if (footer_offset > SIZE_MAX) { + return false; + } + if (footer_offset + AMDUAT_ASL_CORE_INDEX_FOOTER_SIZE != bytes.len) { + return false; + } + + footer.crc64 = amduat_asl_core_index_load_u64_le( + data + (size_t)footer_offset); + footer.seal_snapshot = amduat_asl_core_index_load_u64_le( + data + (size_t)footer_offset + 8); + footer.seal_time_ns = amduat_asl_core_index_load_u64_le( + data + (size_t)footer_offset + 16); + + if (amduat_asl_core_index_crc64(data, (size_t)footer_offset) != + footer.crc64) { + return false; + } + + if (header.bloom_size != 0) { + if (!amduat_octets_clone( + amduat_octets(data + (size_t)header.bloom_offset, + (size_t)header.bloom_size), + &out_segment->bloom)) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + } + + if (!amduat_octets_clone( + amduat_octets(data + (size_t)header.digests_offset, + (size_t)header.digests_size), + &out_segment->digests)) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + + if (record_count != 0) { + out_segment->records = (amduat_asl_index_record_t *)calloc( + record_count, sizeof(*out_segment->records)); + if (out_segment->records == NULL) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + } + out_segment->record_count = record_count; + + if (extent_count != 0) { + out_segment->extents = (amduat_asl_extent_record_t *)calloc( + extent_count, sizeof(*out_segment->extents)); + if (out_segment->extents == NULL) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + } + out_segment->extent_count = extent_count; + + for (i = 0; i < extent_count; ++i) { + size_t base = (size_t)header.extents_offset + + i * AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE; + amduat_asl_extent_record_t *extent = &out_segment->extents[i]; + extent->block_id = amduat_asl_core_index_load_u64_le(data + base); + extent->offset = amduat_asl_core_index_load_u32_le(data + base + 8); + extent->length = amduat_asl_core_index_load_u32_le(data + base + 12); + if (extent->length == 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + } + + max_visibility = 0; + for (i = 0; i < record_count; ++i) { + size_t base = (size_t)header.records_offset + + i * AMDUAT_ASL_CORE_INDEX_RECORD_SIZE; + amduat_asl_index_record_t *record = &out_segment->records[i]; + uint64_t extents_offset; + size_t extent_index; + size_t j; + uint64_t total_len; + bool is_tombstone; + + record->hash_id = amduat_asl_core_index_load_u32_le(data + base); + record->digest_len = amduat_asl_core_index_load_u16_le(data + base + 4); + record->reserved0 = amduat_asl_core_index_load_u16_le(data + base + 6); + record->digest_offset = amduat_asl_core_index_load_u64_le(data + base + 8); + record->extents_offset = amduat_asl_core_index_load_u64_le( + data + base + 16); + record->extent_count = amduat_asl_core_index_load_u32_le(data + base + 24); + record->total_length = amduat_asl_core_index_load_u32_le(data + base + 28); + record->domain_id = amduat_asl_core_index_load_u32_le(data + base + 32); + record->visibility = data[base + 36]; + record->has_cross_domain_source = data[base + 37]; + record->reserved1 = amduat_asl_core_index_load_u16_le(data + base + 38); + record->cross_domain_source = + amduat_asl_core_index_load_u32_le(data + base + 40); + record->flags = amduat_asl_core_index_load_u32_le(data + base + 44); + + if (!legacy_defaults) { + if (record->reserved0 != 0 || record->reserved1 != 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->visibility > 1 || record->has_cross_domain_source > 1) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->has_cross_domain_source == 0 && + record->cross_domain_source != 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + } else { + record->domain_id = 0; + record->visibility = 0; + record->has_cross_domain_source = 0; + record->cross_domain_source = 0; + } + + if ((record->flags & ~AMDUAT_ASL_INDEX_FLAG_TOMBSTONE) != 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->digest_len == 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->digest_len > header.digests_size) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->digest_offset < header.digests_offset) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->digest_offset > + header.digests_offset + header.digests_size - + record->digest_len) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + + is_tombstone = + (record->flags & AMDUAT_ASL_INDEX_FLAG_TOMBSTONE) != 0; + if (is_tombstone) { + if (record->extent_count != 0 || record->total_length != 0 || + record->extents_offset != 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->visibility > max_visibility) { + max_visibility = record->visibility; + } + continue; + } + + if (record->extent_count == 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + extents_offset = record->extents_offset; + if (extents_offset < header.extents_offset) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (!amduat_asl_core_index_is_aligned8(extents_offset)) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (record->extent_count > + UINT64_MAX / AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if ((extents_offset - header.extents_offset) % + AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE != + 0) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + if (extents_offset + (uint64_t)record->extent_count * + AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE > + header.extents_offset + extents_bytes) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + extent_index = (size_t)((extents_offset - header.extents_offset) / + AMDUAT_ASL_CORE_INDEX_EXTENT_SIZE); + total_len = 0; + for (j = 0; j < record->extent_count; ++j) { + total_len += out_segment->extents[extent_index + j].length; + if (total_len > UINT32_MAX) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + } + if ((uint32_t)total_len != record->total_length) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + + if (record->visibility > max_visibility) { + max_visibility = record->visibility; + } + } + + if (!legacy_defaults) { + if (max_visibility != header.segment_visibility) { + amduat_enc_asl_core_index_free(out_segment); + return false; + } + } else { + header.segment_domain_id = 0; + header.segment_visibility = 0; + header.federation_version = 0; + header.reserved0 = 0; + } + + out_segment->header = header; + out_segment->footer = footer; + return true; +} + +void amduat_enc_asl_core_index_free(amduat_asl_core_index_segment_t *segment) { + if (segment == NULL) { + return; + } + free(segment->records); + segment->records = NULL; + segment->record_count = 0; + free(segment->extents); + segment->extents = NULL; + segment->extent_count = 0; + amduat_octets_free(&segment->digests); + amduat_octets_free(&segment->bloom); +} diff --git a/tests/enc/test_asl_core_index.c b/tests/enc/test_asl_core_index.c new file mode 100644 index 0000000..212d631 --- /dev/null +++ b/tests/enc/test_asl_core_index.c @@ -0,0 +1,336 @@ +#include "amduat/enc/asl_core_index.h" + +#include +#include +#include +#include + +static void store_u16_le(uint8_t *out, uint16_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); +} + +static void store_u64_le(uint8_t *out, uint64_t value) { + out[0] = (uint8_t)(value & 0xffu); + out[1] = (uint8_t)((value >> 8) & 0xffu); + out[2] = (uint8_t)((value >> 16) & 0xffu); + out[3] = (uint8_t)((value >> 24) & 0xffu); + out[4] = (uint8_t)((value >> 32) & 0xffu); + out[5] = (uint8_t)((value >> 40) & 0xffu); + out[6] = (uint8_t)((value >> 48) & 0xffu); + out[7] = (uint8_t)((value >> 56) & 0xffu); +} + +static uint64_t load_u64_le(const uint8_t *data) { + return (uint64_t)data[0] | ((uint64_t)data[1] << 8) | + ((uint64_t)data[2] << 16) | ((uint64_t)data[3] << 24) | + ((uint64_t)data[4] << 32) | ((uint64_t)data[5] << 40) | + ((uint64_t)data[6] << 48) | ((uint64_t)data[7] << 56); +} + +static uint64_t crc64_ecma(const uint8_t *data, size_t len) { + uint64_t crc = 0u; + size_t i; + + for (i = 0; i < len; ++i) { + uint64_t bit; + uint8_t value = data[i]; + crc ^= ((uint64_t)value) << 56; + for (bit = 0; bit < 8; ++bit) { + if (crc & 0x8000000000000000ull) { + crc = (crc << 1) ^ 0x42f0e1eba9ea3693ull; + } else { + crc <<= 1; + } + } + } + return crc; +} + +static void update_crc(amduat_octets_t bytes) { + size_t footer_offset = bytes.len - AMDUAT_ASL_CORE_INDEX_FOOTER_SIZE; + uint64_t crc = crc64_ecma(bytes.data, footer_offset); + store_u64_le((uint8_t *)bytes.data + footer_offset, crc); +} + +static void build_simple_segment(amduat_asl_core_index_segment_t *segment, + amduat_asl_index_record_t *record, + amduat_asl_extent_record_t *extent, + uint8_t digest_bytes[8]) { + memset(segment, 0, sizeof(*segment)); + memset(record, 0, sizeof(*record)); + memset(extent, 0, sizeof(*extent)); + + digest_bytes[0] = 0x01; + digest_bytes[1] = 0x02; + digest_bytes[2] = 0x03; + digest_bytes[3] = 0x04; + digest_bytes[4] = 0x05; + digest_bytes[5] = 0x06; + digest_bytes[6] = 0x07; + digest_bytes[7] = 0x08; + + segment->header.snapshot_min = 5; + segment->header.snapshot_max = 6; + segment->header.segment_domain_id = 7; + segment->header.segment_visibility = 1; + segment->header.federation_version = 0; + segment->header.flags = 0; + segment->header.reserved0 = 0; + + record->hash_id = 9; + record->digest_len = 8; + record->extent_count = 1; + record->total_length = 3; + record->domain_id = 7; + record->visibility = 1; + record->has_cross_domain_source = 0; + record->cross_domain_source = 0; + record->flags = 0; + + extent->block_id = 123; + extent->offset = 0; + extent->length = 3; + + segment->records = record; + segment->record_count = 1; + segment->digests = amduat_octets(digest_bytes, 8); + segment->extents = extent; + segment->extent_count = 1; + segment->footer.seal_snapshot = 10; + segment->footer.seal_time_ns = 11; +} + +static int test_round_trip(void) { + amduat_asl_core_index_segment_t segment; + amduat_asl_index_record_t record; + amduat_asl_extent_record_t extent; + uint8_t digest_bytes[8]; + amduat_octets_t encoded; + amduat_asl_core_index_segment_t decoded; + int exit_code = 1; + + build_simple_segment(&segment, &record, &extent, digest_bytes); + + if (!amduat_enc_asl_core_index_encode_v1(&segment, &encoded)) { + fprintf(stderr, "encode failed\n"); + return exit_code; + } + + if (!amduat_enc_asl_core_index_decode_v1(encoded, &decoded)) { + fprintf(stderr, "decode failed\n"); + goto cleanup; + } + + if (decoded.header.version != AMDUAT_ASL_CORE_INDEX_VERSION || + decoded.header.header_size != AMDUAT_ASL_CORE_INDEX_HEADER_SIZE) { + fprintf(stderr, "header mismatch\n"); + goto cleanup_decoded; + } + if (decoded.record_count != 1 || decoded.extent_count != 1) { + fprintf(stderr, "count mismatch\n"); + goto cleanup_decoded; + } + if (decoded.header.segment_domain_id != 7 || + decoded.header.segment_visibility != 1) { + fprintf(stderr, "segment federation mismatch\n"); + goto cleanup_decoded; + } + if (decoded.records[0].hash_id != record.hash_id || + decoded.records[0].digest_len != record.digest_len || + decoded.records[0].total_length != record.total_length || + decoded.records[0].domain_id != record.domain_id || + decoded.records[0].visibility != record.visibility || + decoded.records[0].flags != record.flags) { + fprintf(stderr, "record mismatch\n"); + goto cleanup_decoded; + } + if (decoded.digests.len != sizeof(digest_bytes) || + memcmp(decoded.digests.data, digest_bytes, sizeof(digest_bytes)) != 0) { + fprintf(stderr, "digest mismatch\n"); + goto cleanup_decoded; + } + if (decoded.extents[0].block_id != extent.block_id || + decoded.extents[0].offset != extent.offset || + decoded.extents[0].length != extent.length) { + fprintf(stderr, "extent mismatch\n"); + goto cleanup_decoded; + } + if (decoded.footer.seal_snapshot != segment.footer.seal_snapshot || + decoded.footer.seal_time_ns != segment.footer.seal_time_ns) { + fprintf(stderr, "footer mismatch\n"); + goto cleanup_decoded; + } + + exit_code = 0; + +cleanup_decoded: + amduat_enc_asl_core_index_free(&decoded); +cleanup: + free((void *)encoded.data); + return exit_code; +} + +static int test_crc_mismatch(void) { + amduat_asl_core_index_segment_t segment; + amduat_asl_index_record_t record; + amduat_asl_extent_record_t extent; + uint8_t digest_bytes[8]; + amduat_octets_t encoded; + amduat_asl_core_index_segment_t decoded; + uint64_t digests_offset; + int exit_code = 1; + + build_simple_segment(&segment, &record, &extent, digest_bytes); + + if (!amduat_enc_asl_core_index_encode_v1(&segment, &encoded)) { + fprintf(stderr, "encode failed\n"); + return exit_code; + } + + digests_offset = load_u64_le(encoded.data + 64); + ((uint8_t *)encoded.data)[digests_offset] ^= 0xffu; + + if (amduat_enc_asl_core_index_decode_v1(encoded, &decoded)) { + fprintf(stderr, "decode unexpectedly succeeded\n"); + amduat_enc_asl_core_index_free(&decoded); + goto cleanup; + } + + exit_code = 0; + +cleanup: + free((void *)encoded.data); + return exit_code; +} + +static int test_invalid_offsets(void) { + amduat_asl_core_index_segment_t segment; + amduat_asl_index_record_t record; + amduat_asl_extent_record_t extent; + uint8_t digest_bytes[8]; + amduat_octets_t encoded; + amduat_asl_core_index_segment_t decoded; + int exit_code = 1; + + build_simple_segment(&segment, &record, &extent, digest_bytes); + + if (!amduat_enc_asl_core_index_encode_v1(&segment, &encoded)) { + fprintf(stderr, "encode failed\n"); + return exit_code; + } + + store_u64_le((uint8_t *)encoded.data + 72, 0); + update_crc(encoded); + + if (amduat_enc_asl_core_index_decode_v1(encoded, &decoded)) { + fprintf(stderr, "decode unexpectedly succeeded\n"); + amduat_enc_asl_core_index_free(&decoded); + goto cleanup; + } + + exit_code = 0; + +cleanup: + free((void *)encoded.data); + return exit_code; +} + +static int test_invalid_federation_fields(void) { + amduat_asl_core_index_segment_t segment; + amduat_asl_index_record_t record; + amduat_asl_extent_record_t extent; + uint8_t digest_bytes[8]; + amduat_octets_t encoded; + amduat_asl_core_index_segment_t decoded; + uint64_t records_offset; + int exit_code = 1; + + build_simple_segment(&segment, &record, &extent, digest_bytes); + + if (!amduat_enc_asl_core_index_encode_v1(&segment, &encoded)) { + fprintf(stderr, "encode failed\n"); + return exit_code; + } + + records_offset = load_u64_le(encoded.data + 40); + ((uint8_t *)encoded.data)[records_offset + 36] = 2; + update_crc(encoded); + + if (amduat_enc_asl_core_index_decode_v1(encoded, &decoded)) { + fprintf(stderr, "decode unexpectedly succeeded\n"); + amduat_enc_asl_core_index_free(&decoded); + goto cleanup; + } + + exit_code = 0; + +cleanup: + free((void *)encoded.data); + return exit_code; +} + +static int test_legacy_defaults(void) { + amduat_asl_core_index_segment_t segment; + amduat_asl_index_record_t record; + amduat_asl_extent_record_t extent; + uint8_t digest_bytes[8]; + amduat_octets_t encoded; + amduat_asl_core_index_segment_t decoded; + uint64_t records_offset; + int exit_code = 1; + + build_simple_segment(&segment, &record, &extent, digest_bytes); + + if (!amduat_enc_asl_core_index_encode_v1(&segment, &encoded)) { + fprintf(stderr, "encode failed\n"); + return exit_code; + } + + store_u16_le((uint8_t *)encoded.data + 8, 2); + ((uint8_t *)encoded.data)[100] = 1; + records_offset = load_u64_le(encoded.data + 40); + ((uint8_t *)encoded.data)[records_offset + 36] = 1; + update_crc(encoded); + + if (!amduat_enc_asl_core_index_decode_v1(encoded, &decoded)) { + fprintf(stderr, "decode failed\n"); + goto cleanup; + } + + if (decoded.header.segment_visibility != 0 || + decoded.records[0].visibility != 0 || + decoded.records[0].domain_id != 0 || + decoded.records[0].has_cross_domain_source != 0 || + decoded.records[0].cross_domain_source != 0) { + fprintf(stderr, "legacy defaults not applied\n"); + goto cleanup_decoded; + } + + exit_code = 0; + +cleanup_decoded: + amduat_enc_asl_core_index_free(&decoded); +cleanup: + free((void *)encoded.data); + return exit_code; +} + +int main(void) { + if (test_round_trip() != 0) { + return 1; + } + if (test_crc_mismatch() != 0) { + return 1; + } + if (test_invalid_offsets() != 0) { + return 1; + } + if (test_invalid_federation_fields() != 0) { + return 1; + } + if (test_legacy_defaults() != 0) { + return 1; + } + return 0; +}