materialize
materialize copied to clipboard
MaterializeInc
thread 'coordinator' panicked at src/adapter/src/catalog.rs: all dropped_notices entries have `Arc::strong_count(_) == 1
What version of Materialize are you using?
97ff6eb84ad1d04759a4b69100f469a420461fa4
What is the issue?
SQLancer QueryPartitioning in builds/6270:
$ docker compose run --detach sqlancer --random-seed 2056923865 --host materialized --port 6875 --username materialize --timeout-seconds 1500 --num-tries 100000 --num-threads 4 --qpg-enable True --random-string-generation ALPHANUMERIC_SPECIALCHAR materialize --oracle QUERY_PARTITIONING
sqlancer-materialized-1 | environmentd: 2024-02-01T08:30:42.419801Z INFO message_command: mz_adapter::catalog::state: drop index database3.public.i0 (u21)
sqlancer-materialized-1 | thread 'coordinator' panicked at src/adapter/src/catalog.rs:293:9:
sqlancer-materialized-1 | all dropped_notices entries have `Arc::strong_count(_) == 1`
sqlancer-materialized-1 | stack backtrace:
sqlancer-materialized-1 | 0: rust_begin_unwind
sqlancer-materialized-1 | at ./rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/std/src/panicking.rs:597:5
sqlancer-materialized-1 | 1: core::panicking::panic_fmt
sqlancer-materialized-1 | at ./rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/panicking.rs:72:14
sqlancer-materialized-1 | 2: <mz_adapter::catalog::Catalog>::drop_plans_and_metainfos
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/catalog.rs:293:9
sqlancer-materialized-1 | 3: <mz_adapter::catalog::Catalog>::transact::<<mz_adapter::coord::Coordinator>::catalog_transact_with<<mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}::{closure#0}::{closure#1}, ()>::{closure#0}::{closure#0}::{closure#10}, ()>::{closure#0}::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/catalog.rs:1248:31
sqlancer-materialized-1 | 4: <mz_adapter::catalog::Catalog>::transact::<<mz_adapter::coord::Coordinator>::catalog_transact_with<<mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}::{closure#0}::{closure#1}, ()>::{closure#0}::{closure#0}::{closure#10}, ()>::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/catalog.rs:1177:5
sqlancer-materialized-1 | 5: <mz_adapter::coord::Coordinator>::catalog_transact_with::<<mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}::{closure#0}::{closure#1}, ()>::{closure#0}::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/ddl.rs:520:14
sqlancer-materialized-1 | 6: <mz_adapter::coord::Coordinator>::catalog_transact_with::<<mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}::{closure#0}::{closure#1}, ()>::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/ddl.rs:192:5
sqlancer-materialized-1 | 7: <mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/ddl.rs:81:14
sqlancer-materialized-1 | 8: <mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/ddl.rs:73:5
sqlancer-materialized-1 | 9: <mz_adapter::coord::Coordinator>::sequence_drop_objects::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/sequencer/inner.rs:895:51
sqlancer-materialized-1 | 10: <mz_adapter::coord::Coordinator>::sequence_plan::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/sequencer.rs:228:86
sqlancer-materialized-1 | 11: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::sequence_plan::{closure#0}> as core::future::future::Future>::poll
sqlancer-materialized-1 | at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
sqlancer-materialized-1 | 12: <core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>> as core::future::future::Future>::poll
sqlancer-materialized-1 | at ./rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/future/future.rs:125:9
sqlancer-materialized-1 | 13: <mz_adapter::coord::Coordinator>::handle_execute_inner::{closure#0}::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/command_handler.rs:771:73
sqlancer-materialized-1 | 14: <mz_adapter::coord::Coordinator>::handle_execute_inner::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/command_handler.rs:449:5
sqlancer-materialized-1 | 15: <mz_adapter::coord::Coordinator>::handle_execute::{closure#0}::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/command_handler.rs:446:54
sqlancer-materialized-1 | 16: <mz_adapter::coord::Coordinator>::handle_execute::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/command_handler.rs:347:5
sqlancer-materialized-1 | 17: <mz_adapter::coord::Coordinator>::handle_command::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/command_handler.rs:113:26
sqlancer-materialized-1 | 18: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::handle_command::{closure#0}> as core::future::future::Future>::poll
sqlancer-materialized-1 | at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
sqlancer-materialized-1 | 19: <core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>> as core::future::future::Future>::poll
sqlancer-materialized-1 | at ./rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/future/future.rs:125:9
sqlancer-materialized-1 | 20: <mz_adapter::coord::Coordinator>::message_command::{closure#0}::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/message_handler.rs:344:34
sqlancer-materialized-1 | 21: <mz_adapter::coord::Coordinator>::message_command::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/message_handler.rs:341:5
sqlancer-materialized-1 | 22: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::message_command::{closure#0}> as core::future::future::Future>::poll
sqlancer-materialized-1 | at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
sqlancer-materialized-1 | 23: <mz_adapter::coord::Coordinator>::handle_message::{closure#0}
sqlancer-materialized-1 | at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0e76f3cb7ce326512-1/materialize/tests/src/adapter/src/coord/message_handler.rs:70:64
sqlancer-materialized-1 | 24: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::handle_message::{closure#0}> as core::future::future::Future>::poll
sqlancer-materialized-1 | at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
sqlancer-materialized-1 | 25: <core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>> as core::future::future::Future>::poll
sqlancer-materialized-1 | at ./rustc/79e9716c980570bfd1f666e3b16ac583f0168962/library/core/src/future/future.rs:125:9
ci-regexp: all dropped_notices entries (should )?have
I tried running the queries from SQLancer but that didn't cause the panic: https://gist.github.com/def-/67bb918263ff52d71ddcffb8fd429041
It seems that my notice cleanup logic fails to clean up everything in some extreme situations. By itself, this should not be very urgent (we should at most spent a few more KBs of memory in every release cycle).
It would be interesting to track if this happens only on ARM, or also on Intel machines.
![sentry[bot] avatar](https://avatars.githubusercontent.com/in/12637?v=4 "Published by a pub.dev verified publisher"){kind=small}
This is now happening in production on 0.85.2! ^ So definitely not ARM-specific. Edit: This was just logged, didn't cause a panic.
From SQLancer PQS: https://buildkite.com/materialize/nightlies/builds/6368#018d8b33-d151-4fbe-9beb-8c4da858503c
sqlancer-materialized-1 | environmentd: 2024-02-08T23:36:15.603145Z INFO message_command:coord::handle_execute{session="57ddbed7-8de3-40ed-b710-5d893eae103f"}:coord::handle_execute_inner{stmt="CREATE TABLE IF NOT EXISTS t0 (c0 bool)"}:sequence_create_table:coord::catalog_transact_with_side_effects:coord::catalog_transact_with:catalog::transact:catalog::transact_inner: mz_adapter::catalog::state: create table database1.public.t0 (u29)
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:15.645090Z INFO mz_persist_client::internal::machine: snapshot sf0925e8b-7c2d-48e1-a1e8-2dea4a71003e as of [1707435371577] now available
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:15.645317Z INFO mz_persist_client::internal::machine: snapshot sf0925e8b-7c2d-48e1-a1e8-2dea4a71003e as of [1707435372633] now available
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:15.791519Z INFO mz_persist_client::internal::machine: snapshot s5637f98c-edd3-4556-8d30-30e1c5cec831 as of [1707435372633] not yet available for v2062 upper [1707435367567]
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:15.807981Z INFO mz_persist_client::internal::machine: snapshot saab25a5a-2f0b-4396-b045-3af27ac9909f as of [1707435371577] now available
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:15.808082Z INFO mz_persist_client::internal::machine: snapshot saab25a5a-2f0b-4396-b045-3af27ac9909f as of [1707435372633] now available
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:16.175146Z INFO mz_persist_client::internal::machine: snapshot sf77ef0f7-ac35-4ba7-9032-28937c23c677 as of [1707435372633] now available
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:16.208879Z INFO mz_persist_client::internal::machine: snapshot s5637f98c-edd3-4556-8d30-30e1c5cec831 as of [1707435371577] now available
sqlancer-materialized-1 | cluster-u1-replica-u1: 2024-02-08T23:36:16.208942Z INFO mz_persist_client::internal::machine: snapshot s5637f98c-edd3-4556-8d30-30e1c5cec831 as of [1707435372633] now available
sqlancer-materialized-1 | environmentd: 2024-02-08T23:36:16.493725Z INFO message_command:coord::handle_execute{session="e6561875-8dc0-4d73-b18b-b1d7b1442890"}:coord::handle_execute_inner{stmt="DROP DATABASE IF EXISTS database0"}:sequence_drop_objects:coord::catalog_transact:coord::catalog_transact_with:catalog::transact:catalog::transact_inner: mz_adapter::catalog::state: drop index database0.public.i18 (u25)
sqlancer-materialized-1 | environmentd: 2024-02-08T23:36:16.493981Z INFO message_command:coord::handle_execute{session="e6561875-8dc0-4d73-b18b-b1d7b1442890"}:coord::handle_execute_inner{stmt="DROP DATABASE IF EXISTS database0"}:sequence_drop_objects:coord::catalog_transact:coord::catalog_transact_with:catalog::transact:catalog::transact_inner: mz_adapter::catalog::state: drop index database0.public.i33 (u24)
sqlancer-materialized-1 | environmentd: 2024-02-08T23:36:16.494183Z INFO message_command:coord::handle_execute{session="e6561875-8dc0-4d73-b18b-b1d7b1442890"}:coord::handle_execute_inner{stmt="DROP DATABASE IF EXISTS database0"}:sequence_drop_objects:coord::catalog_transact:coord::catalog_transact_with:catalog::transact:catalog::transact_inner: mz_adapter::catalog::state: drop table database0.public.t0 (u4)
sqlancer-materialized-1 | environmentd: 2024-02-08T23:36:16.494336Z INFO message_command:coord::handle_execute{session="e6561875-8dc0-4d73-b18b-b1d7b1442890"}:coord::handle_execute_inner{stmt="DROP DATABASE IF EXISTS database0"}:sequence_drop_objects:coord::catalog_transact:coord::catalog_transact_with:catalog::transact:catalog::transact_inner: mz_adapter::catalog::state: drop table database0.public.t1 (u8)
sqlancer-materialized-1 | environmentd: 2024-02-08T23:36:16.494474Z INFO message_command:coord::handle_execute{session="e6561875-8dc0-4d73-b18b-b1d7b1442890"}:coord::handle_execute_inner{stmt="DROP DATABASE IF EXISTS database0"}:sequence_drop_objects:coord::catalog_transact:coord::catalog_transact_with:catalog::transact:catalog::transact_inner: mz_adapter::catalog::state: drop table database0.public.t2 (u12)
sqlancer-materialized-1 | environmentd: 2024-02-08T23:36:16.494663Z INFO message_command:coord::handle_execute{session="e6561875-8dc0-4d73-b18b-b1d7b1442890"}:coord::handle_execute_inner{stmt="DROP DATABASE IF EXISTS database0"}:sequence_drop_objects:coord::catalog_transact:coord::catalog_transact_with:catalog::transact:catalog::transact_inner: mz_adapter::catalog::state: drop table database0.public.t3 (u14)
sqlancer-materialized-1 | thread 'coordinator' panicked at src/adapter/src/catalog.rs:306:13:
sqlancer-materialized-1 | all dropped_notices entries have `Arc::strong_count(_) == 1`; bad_notices = {(id = t940, kind = IndexAlreadyExists, deps = {User(14), User(24)}, strong_count = 4)}; drop_ids = {User(4), User(8), User(12), User(14), User(24), User(25)}
I have extracted a timeline of the involved objects from the above, but I cannot reproduce it locally. It seems to be some concurrency issue. @def- does SQLancer run its queries from different clients / sessions? In particular, is it possible that the two indexes were created by different sessions around the same time?
Timeline
-- 2024-02-08T23:22:11.271681Z
-- create schema database0.public
CREATE DATABASE database0;
SET database = database0;
-- 2024-02-08T23:22:18.819435Z
-- 6a7ed82b-1183-41f2-8899-c96800796f1f
-- create table database0.public.t0 (u4)
CREATE TABLE IF NOT EXISTS t0 (c0 bool);
-- 2024-02-08T23:22:33.725777Z
-- a732c905-d8dc-45aa-8847-e644af31de00
-- create table database0.public.t1 (u8)
CREATE TABLE t1 (c0 bool);
-- 2024-02-08T23:23:01.368804Z
-- a732c905-d8dc-45aa-8847-e644af31de00
-- create table database0.public.t2 (u12)
CREATE TABLE t2 (c0 bool);
-- 2024-02-08T23:23:37.714427Z
-- a732c905-d8dc-45aa-8847-e644af31de00
-- create table database0.public.t3 (u14)
CREATE TABLE t3 (c0 int2);
-- 2024-02-08T23:28:23.805426Z
-- ???
-- create index database0.public.i33 (u24)
CREATE DEfAULT INDEX i33 ON t3;
-- 2024-02-08T23:28:24.610971Z
-- ???
-- create index database0.public.i18 (u25)
CREATE DEfAULT INDEX i18 ON t3;
-- 2024-02-08T23:36:16.493725Z
-- drop everything
-- e6561875-8dc0-4d73-b18b-b1d7b1442890
DROP DATABASE IF EXISTS database0;
Yes, SQLancer runs with 4 parallel threads in our testing by default. But that means there are 4 separate databases (CREATE DATABASE, not Mz instances), which don't interact with each other at all.
Happened again in Sentry with more information:
all dropped_notices entries have `Arc::strong_count(_) == 1`; bad_notices = {(id = t7843026, kind = IndexTooWideForLiteralConstraints, deps = {User(63911), User(63912)}, strong_count = 4), (id = t7843133, kind = IndexTooWideForLiteralConstraints, deps = {User(63911), User(63912)}, strong_count = 4), (id = t7843300, kind = IndexTooWideForLiteralConstraints, deps = {User(63911), User(63912)}, strong_count = 4)}; drop_ids = {User(63857), User(63858), User(63907), User(63908), User(63911), User(63912), User(63923), User(63924), User(63927), User(63928), User(63931), User(63932)}
Further occurrence in an instance classified as production: https://materializeinc.sentry.io/issues/5023826865/?project=6780145
I'll try to find some time in the next weeks to rework the logic so it is based on GlobalId references and not on Arc<OptimizerNotice> objects.
No. @ggevay is going to be adding more notices in the coming weeks and I will discuss with him—maybe he has ideas how to troubleshoot this.
I want to re-iterate that observing this soft_assert is only indicative of a memory leaks that is in the order of KiB * O(n) where n is the number of catalog objects producing notices within a single week. So far we have not seen any other evidence of environmentd leaking enough memory to warrant allocating immediate cycles for investigating this.
If we see such evidence feel free to drop a notice in this issue and I will discuss with @lfest to put this on the roadmap. The importance of fixing this might change once we start rolling more notices (especially notices that are more likely to occur for catalog items).
@aalexandrov: Thanks for the update and the information about the severity!
Here are all augmented error logs that can be seen in Sentry:
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t10480680, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 4),
(id = t10480687, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 4)
};
drop_ids = {u12273, u12274, u12342, u12343, u12344, u12345}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t10480691, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 4),
(id = t10480701, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 4),
(id = t10480743, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 4),
(id = t10480759, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 4)
};
drop_ids = {u12279, u12280, u12329, u12330, u12331, u12332, u12335, u12336, u12346, u12347, u12348, u12349, u12354, u12355, u12356, u12357}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t10480726, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 7),
(id = t10480733, kind = IndexTooWideForLiteralConstraints, deps = {u12335, u12336}, strong_count = 7)
};
drop_ids = {u12275, u12276, u12350, u12351, u12352, u12353}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t11015812, kind = IndexTooWideForLiteralConstraints, deps = {u13174, u13175}, strong_count = 7),
(id = t11015825, kind = IndexTooWideForLiteralConstraints, deps = {u13174, u13175}, strong_count = 7)
};
drop_ids = {u12948, u12949, u13273, u13276, u13277, u13279}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t11428385, kind = IndexTooWideForLiteralConstraints, deps = {u13888, u13889}, strong_count = 10),
(id = t11428398, kind = IndexTooWideForLiteralConstraints, deps = {u13888, u13889}, strong_count = 10),
(id = t11428489, kind = IndexTooWideForLiteralConstraints, deps = {u13888, u13889}, strong_count = 10),
(id = t11428499, kind = IndexTooWideForLiteralConstraints, deps = {u13888, u13889}, strong_count = 10),
(id = t11428568, kind = IndexTooWideForLiteralConstraints, deps = {u13888, u13889}, strong_count = 10),
(id = t11428599, kind = IndexTooWideForLiteralConstraints, deps = {u13888, u13889}, strong_count = 10)
};
drop_ids = {u13659, u13660, u13873, u13876, u13878, u13879, u13888, u13889, u13953, u13954, u13957, u13958, u13968, u13970, u13972, u13973, u13983, u13984, u13985, u13987}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t11432882, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4),
(id = t11432913, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4),
(id = t11432970, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4),
(id = t11432995, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4),
(id = t11433034, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4),
(id = t11433050, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4)
};
drop_ids = {u14103, u14107, u14282, u14285, u14287, u14289, u14295, u14297, u14351, u14354, u14357, u14358, u14369, u14370, u14371, u14374, u14379, u14380, u14381, u14382}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t11432935, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4),
(id = t11432951, kind = IndexTooWideForLiteralConstraints, deps = {u14295, u14297}, strong_count = 4)
};
drop_ids = {u14087, u14090, u14361, u14362, u14365, u14366}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t11554838, kind = IndexTooWideForLiteralConstraints, deps = {u17127, u17129}, strong_count = 4),
(id = t11554851, kind = IndexTooWideForLiteralConstraints, deps = {u17127, u17129}, strong_count = 4)
};
drop_ids = {u16904, u16905, u17203, u17205, u17208, u17209}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t2757551, kind = IndexTooWideForLiteralConstraints, deps = {u7899, u7900}, strong_count = 4),
(id = t2757602, kind = IndexTooWideForLiteralConstraints, deps = {u7899, u7900}, strong_count = 4)
};
drop_ids = {u7895, u7896, u7899, u7900, u7910, u7911, u7912, u7913, u7914, u7915, u7920, u7923, u7924, u7925, u7926, u7927, u7928}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t2879275, kind = IndexTooWideForLiteralConstraints, deps = {u8515, u8516}, strong_count = 4),
(id = t2879416, kind = IndexTooWideForLiteralConstraints, deps = {u8515, u8516}, strong_count = 4)
};
drop_ids = {u8511, u8512, u8515, u8516, u8526, u8527, u8528, u8529, u8530, u8531, u8536, u8539, u8540, u8541, u8542, u8543, u8544}
all dropped_notices entries have `Arc::strong_count(_) == 1`;
bad_notices = {
(id = t7843026, kind = IndexTooWideForLiteralConstraints, deps = {u63911, u63912}, strong_count = 4),
(id = t7843133, kind = IndexTooWideForLiteralConstraints, deps = {u63911, u63912}, strong_count = 4),
(id = t7843300, kind = IndexTooWideForLiteralConstraints, deps = {u63911, u63912}, strong_count = 4)
};
drop_ids = {u63857, u63858, u63907, u63908, u63911, u63912, u63923, u63924, u63927, u63928, u63931, u63932}
One thing to note is that this only seems to appear for IndexTooWideForLiteralConstraints,
Occurred again in Parallel Workload (rename):
thread 'coordinator' panicked at src/adapter/src/catalog.rs:297:13:
all dropped_notices entries have `Arc::strong_count(_) == 1`; bad_notices = {(id = t856, kind = IndexAlreadyExists, deps = {User(32), User(206)}, strong_count = 4)}; drop_ids = {User(32), User(206), User(308)}
stack backtrace:
0: rust_begin_unwind
at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/std/src/panicking.rs:652:5
1: core::panicking::panic_fmt
at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/panicking.rs:72:14
2: <mz_adapter::catalog::Catalog>::drop_plans_and_metainfos
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/catalog.rs:297:13
3: <mz_adapter::catalog::Catalog>::transact::{closure#0}::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/catalog/transact.rs:421:31
4: <tracing::instrument::Instrumented<<mz_adapter::catalog::Catalog>::transact::{closure#0}::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
5: <mz_adapter::catalog::Catalog>::transact::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/catalog/transact.rs:340:5
6: <mz_adapter::coord::Coordinator>::catalog_transact_inner::{closure#0}::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/ddl.rs:517:14
7: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::catalog_transact_inner::{closure#0}::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
8: <mz_adapter::coord::Coordinator>::catalog_transact_inner::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/ddl.rs:182:5
9: <mz_adapter::coord::Coordinator>::catalog_transact_conn::{closure#0}::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/ddl.rs:120:71
10: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::catalog_transact_conn::{closure#0}::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
11: <mz_adapter::coord::Coordinator>::catalog_transact_conn::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/ddl.rs:114:5
12: <mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/ddl.rs:77:14
13: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
14: <mz_adapter::coord::Coordinator>::catalog_transact::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/ddl.rs:70:5
15: <mz_adapter::coord::Coordinator>::sequence_drop_objects::{closure#0}::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/sequencer/inner.rs:1186:51
16: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::sequence_drop_objects::{closure#0}::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
17: <mz_adapter::coord::Coordinator>::sequence_drop_objects::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/sequencer/inner.rs:1152:5
18: <mz_adapter::coord::Coordinator>::sequence_plan::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/sequencer.rs:232:86
19: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::sequence_plan::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
20: <core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>> as core::future::future::Future>::poll
at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
21: <mz_adapter::coord::Coordinator>::handle_execute_inner::{closure#0}::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/command_handler.rs:947:69
22: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::handle_execute_inner::{closure#0}::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
23: <mz_adapter::coord::Coordinator>::handle_execute_inner::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/command_handler.rs:503:5
24: <mz_adapter::coord::Coordinator>::handle_deferred_statement::{closure#0}::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/sequencer/inner.rs:4528:62
25: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::handle_deferred_statement::{closure#0}::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
26: <mz_adapter::coord::Coordinator>::handle_deferred_statement::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/sequencer/inner.rs:4518:5
27: <mz_adapter::coord::Coordinator>::handle_message::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord/message_handler.rs:235:54
28: <tracing::instrument::Instrumented<<mz_adapter::coord::Coordinator>::handle_message::{closure#0}> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
29: <core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>> as core::future::future::Future>::poll
at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
30: <mz_adapter::coord::Coordinator>::serve::{closure#0}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord.rs:3076:48
31: <core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>> as core::future::future::Future>::poll
at ./rustc/129f3b9964af4d4a709d1383930ade12dfe7c081/library/core/src/future/future.rs:123:9
32: <tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>> as core::future::future::Future>::poll
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.37/src/instrument.rs:272:9
33: <tokio::runtime::park::CachedParkThread>::block_on::<tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>>::{closure#0}
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/park.rs:281:63
34: tokio::runtime::coop::with_budget::<core::task::poll::Poll<()>, <tokio::runtime::park::CachedParkThread>::block_on<tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>>::{closure#0}>
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:107:5
35: tokio::runtime::coop::budget::<core::task::poll::Poll<()>, <tokio::runtime::park::CachedParkThread>::block_on<tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>>::{closure#0}>
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/coop.rs:73:5
36: <tokio::runtime::park::CachedParkThread>::block_on::<tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>>
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/park.rs:281:31
37: <tokio::runtime::context::blocking::BlockingRegionGuard>::block_on::<tracing::instrument::Instrumented<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>>
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/blocking.rs:66:9
38: <tokio::runtime::handle::Handle>::block_on::<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>::{closure#0}
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/handle.rs:310:22
39: tokio::runtime::context::runtime::enter_runtime::<<tokio::runtime::handle::Handle>::block_on<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>::{closure#0}, ()>
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/context/runtime.rs:65:16
40: <tokio::runtime::handle::Handle>::block_on::<core::pin::Pin<alloc::boxed::Box<dyn core::future::future::Future<Output = ()>>>>
at ./cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.38.0/src/runtime/handle.rs:309:9
41: mz_adapter::coord::serve::{closure#0}::{closure#2}
at ./var/lib/buildkite-agent/builds/buildkite-builders-aarch64-585fc7f-i-0299b997aec811a35-1/materialize/test/src/adapter/src/coord.rs:3713:21
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.