start_cluster = <helpers.cluster.ClickHouseCluster object at 0x7fae3b17cf40>

    def test_mutate_and_upgrade(start_cluster):
        for node in [node1, node2]:
            node.query("DROP TABLE IF EXISTS mt")
            node.query(
                "CREATE TABLE mt (EventDate Date, id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple()".format(
                    node.name
                )
            )
    
        node1.query("INSERT INTO mt VALUES ('2020-02-13', 1), ('2020-02-13', 2);")
    
        node1.query("ALTER TABLE mt DELETE WHERE id = 2", settings={"mutations_sync": "2"})
        node2.query("SYSTEM SYNC REPLICA mt", timeout=15)
    
        node2.query("DETACH TABLE mt")  # stop being leader
        node1.query("DETACH TABLE mt")  # stop being leader
        node1.restart_with_latest_version(signal=9, fix_metadata=True)
>       node2.restart_with_latest_version(signal=9, fix_metadata=True)

test_version_update_after_mutation/test.py:62: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
helpers/cluster.py:3697: in restart_with_latest_version
    self.wait_start(time_left)

2023.01.04 21:17:22.151236 [ 142 ] {} <Error> Application: Caught exception while loading metadata: Code: 219. DB::Exception: Cannot drop: filesystem error: in remove: Directory not empty ["/var/lib/clickhouse/data
/system/"]. Probably database contain some detached tables or metadata leftovers from Ordinary engine. If you want to remove all data anyway, try to attach database back and drop it again with enabled force_remove_
data_recursively_on_drop setting: Exception while trying to convert database system from Ordinary to Atomic. It may be in some intermediate state. You can finish conversion manually by moving the rest tables from s
ystem to .tmp_convert.system.16495121816668103290 (using RENAME TABLE) and executing DROP DATABASE system and RENAME DATABASE .tmp_convert.system.16495121816668103290 TO system. (DATABASE_NOT_EMPTY), Stack trace (w
hen copying this message, always include the lines below):

0. DB::Exception::Exception(DB::Exception::MessageMasked const&, int, bool) @ 0xf2d0b7a in /usr/bin/clickhouse
1. ? @ 0xde1842e in /usr/bin/clickhouse
2. DB::DatabaseOnDisk::drop(std::__1::shared_ptr<DB::Context const>) @ 0x13dc7e1f in /usr/bin/clickhouse
3. DB::DatabaseCatalog::detachDatabase(std::__1::shared_ptr<DB::Context const>, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, bool, bool) @ 0x140687f4 in /usr/bin/clic
khouse
4. DB::InterpreterDropQuery::executeToDatabaseImpl(DB::ASTDropQuery const&, std::__1::shared_ptr<DB::IDatabase>&, std::__1::vector<StrongTypedef<wide::integer<128ul, unsigned int>, DB::UUIDTag>, std::__1::allocator
<StrongTypedef<wide::integer<128ul, unsigned int>, DB::UUIDTag>>>&) @ 0x148657eb in /usr/bin/clickhouse
5. DB::InterpreterDropQuery::executeToDatabase(DB::ASTDropQuery const&) @ 0x148621bd in /usr/bin/clickhouse
6. DB::InterpreterDropQuery::execute() @ 0x1486155c in /usr/bin/clickhouse
7. ? @ 0x14d417b3 in /usr/bin/clickhouse
8. DB::executeQuery(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>> const&, std::__1::shared_ptr<DB::Context>, bool, DB::QueryProcessingStage::Enum) @ 0x14d3e86d in /usr/bin/cli
ckhouse
9. ? @ 0x14d657fe in /usr/bin/clickhouse
10. DB::maybeConvertSystemDatabase(std::__1::shared_ptr<DB::Context>) @ 0x14d6372f in /usr/bin/clickhouse
11. DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char>>, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::al
locator<char>>>> const&) @ 0xf360de1 in /usr/bin/clickhouse
12. Poco::Util::Application::run() @ 0x188bb662 in /usr/bin/clickhouse
13. DB::Server::run() @ 0xf350d79 in /usr/bin/clickhouse
14. Poco::Util::ServerApplication::run(int, char**) @ 0x188cf9ba in /usr/bin/clickhouse
15. mainEntryClickHouseServer(int, char**) @ 0xf34d7de in /usr/bin/clickhouse
16. main @ 0x89c1f73 in /usr/bin/clickhouse
17. __libc_start_main @ 0x21b97 in /lib/x86_64-linux-gnu/libc-2.27.so
18. _start @ 0x7f39a2e in /usr/bin/clickhouse
 (version 22.13.1.1257)
2023.01.04 21:17:22.202766 [ 142 ] {} <Information> Application: Shutting down storages.