____________________________ test_cgroup_cpu_limit _____________________________
[gw0] linux -- Python 3.8.10 /usr/bin/python3

    def test_cgroup_cpu_limit():
        for num_cpus in (1, 2, 4, 2.8):
>           result = run_with_cpu_limit(
                "clickhouse local -q \"select value from system.settings where name='max_threads'\"",
                num_cpus,
            )

test_cgroup_limit/test.py:43: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
test_cgroup_limit/test.py:38: in run_with_cpu_limit
    return run_command_in_container(cmd, *args)
test_cgroup_limit/test.py:19: in run_command_in_container
    return subprocess.check_output(
/usr/lib/python3.8/subprocess.py:415: in check_output
    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

input = None, capture_output = False, timeout = None, check = True
popenargs = (['docker', 'run', '--rm', '--cpus', '1', '--volume', ...],)
kwargs = {'stdout': -1}, process = <subprocess.Popen object at 0x7f3ed7d9c730>
stdout = b'', stderr = None, retcode = 125

    def run(*popenargs,
            input=None, capture_output=False, timeout=None, check=False, **kwargs):
        """Run command with arguments and return a CompletedProcess instance.

        The returned instance will have attributes args, returncode, stdout and
        stderr. By default, stdout and stderr are not captured, and those attributes
        will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.

        If check is True and the exit code was non-zero, it raises a
        CalledProcessError. The CalledProcessError object will have the return code
        in the returncode attribute, and output & stderr attributes if those streams
        were captured.

        If timeout is given, and the process takes too long, a TimeoutExpired
        exception will be raised.

        There is an optional argument "input", allowing you to
        pass bytes or a string to the subprocess's stdin.  If you use this argument
        you may not also use the Popen constructor's "stdin" argument, as
        it will be used internally.

        By default, all communication is in bytes, and therefore any "input" should
        be bytes, and the stdout and stderr will be bytes. If in text mode, any
        "input" should be a string, and stdout and stderr will be strings decoded
        according to locale encoding, or by "encoding" if set. Text mode is
        triggered by setting any of text, encoding, errors or universal_newlines.

        The other arguments are the same as for the Popen constructor.
        """
        if input is not None:
            if kwargs.get('stdin') is not None:
                raise ValueError('stdin and input arguments may not both be used.')
            kwargs['stdin'] = PIPE

        if capture_output:
            if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None:
                raise ValueError('stdout and stderr arguments may not be used '
                                 'with capture_output.')
            kwargs['stdout'] = PIPE
            kwargs['stderr'] = PIPE

        with Popen(*popenargs, **kwargs) as process:
            try:
                stdout, stderr = process.communicate(input, timeout=timeout)
            except TimeoutExpired as exc:
                process.kill()
                if _mswindows:
                    # Windows accumulates the output in a single blocking
                    # read() call run on child threads, with the timeout
                    # being done in a join() on those threads.  communicate()
                    # _after_ kill() is required to collect that and add it
                    # to the exception.
                    exc.stdout, exc.stderr = process.communicate()
                else:
                    # POSIX _communicate already populated the output so
                    # far into the TimeoutExpired exception.
                    process.wait()
                raise
            except:  # Including KeyboardInterrupt, communicate handled that.
                process.kill()
                # We don't call process.wait() as .__exit__ does that for us.
                raise
            retcode = process.poll()
            if check and retcode:
>               raise CalledProcessError(retcode, process.args,
                                         output=stdout, stderr=stderr)
E               subprocess.CalledProcessError: Command '['docker', 'run', '--rm', '--cpus', '1', '--volume', '/clickhouse:/usr/bin/clickhouse', 'ubuntu:20.04', 'sh', '-c', 'clickhouse local -q "select value from system.settings where name=\'max_threads\'"']' returned non-zero exit status 125.

/usr/lib/python3.8/subprocess.py:516: CalledProcessError

____________ test_table_in_replicated_database_with_not_synced_def _____________
[gw2] linux -- Python 3.8.10 /usr/bin/python3

    def test_table_in_replicated_database_with_not_synced_def():
        node1.query(
            "CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}')"
        )

        node1.query(
            "CREATE TABLE mydb.tbl (x UInt8, y String) ENGINE=ReplicatedMergeTree ORDER BY tuple()"
        )

        node1.query("ALTER TABLE mydb.tbl MODIFY COLUMN x String")

        backup_name = new_backup_name()
        node2.query(f"BACKUP DATABASE mydb ON CLUSTER 'cluster' TO {backup_name}")

        node1.query("DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY")

        # But synced after RESTORE anyway
>       node1.query(
            f"RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM {backup_name} SETTINGS replica_num_in_backup=1"
        )

test_backup_restore_on_cluster/test.py:722: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
helpers/cluster.py:3055: in query
    return self.client.query(
helpers/client.py:24: in wrap
    return func(self, *args, **kwargs)
helpers/client.py:51: in query
    return self.get_query_request(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <helpers.client.CommandRequest object at 0x7f15841ac190>

    def get_answer(self):
        self.process.wait(timeout=DEFAULT_QUERY_TIMEOUT)
        self.stdout_file.seek(0)
        self.stderr_file.seek(0)

        stdout = self.stdout_file.read().decode("utf-8", errors="replace")
        stderr = self.stderr_file.read().decode("utf-8", errors="replace")

        if (
            self.timer is not None
            and not self.process_finished_before_timeout
            and not self.ignore_error
        ):
            logging.debug(f"Timed out. Last stdout:{stdout}, stderr:{stderr}")
            raise QueryTimeoutExceedException("Client timed out!")

        if (self.process.returncode != 0 or stderr) and not self.ignore_error:
>           raise QueryRuntimeException(
                "Client failed! Return code: {}, stderr: {}".format(
                    self.process.returncode, stderr
                ),
                self.process.returncode,
                stderr,
            )
E           helpers.client.QueryRuntimeException: Client failed! Return code: 96, stderr: Received exception from server (version 22.8.20):
E           Code: 608. DB::Exception: Received from 172.16.7.5:9000. DB::Exception: Got error from node1:9000. DB::Exception: Couldn't restore table mydb.tbl on other node or sync it (elapsed 1.003869s): While creating table mydb.tbl. Stack trace:
E           
E           0. DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0xa40495a in /usr/bin/clickhouse
E           1. DB::Exception::Exception<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >(int, fmt::v8::basic_format_string<char, fmt::v8::type_identity<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >::type, fmt::v8::type_identity<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >::type, fmt::v8::type_identity<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >::type>, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >&&) @ 0xcb245ec in /usr/bin/clickhouse
E           2. DB::DatabaseReplicated::createTableRestoredFromBackup(std::__1::shared_ptr<DB::IAST> const&, std::__1::shared_ptr<DB::Context>, std::__1::shared_ptr<DB::IRestoreCoordination>, unsigned long) @ 0x143615fe in /usr/bin/clickhouse
E           3. DB::RestorerFromBackup::createTable(DB::QualifiedTableName const&) @ 0x13f282a0 in /usr/bin/clickhouse
E           4. DB::RestorerFromBackup::createTables() @ 0x13f12b3b in /usr/bin/clickhouse
E           5. DB::RestorerFromBackup::run(DB::RestorerFromBackup::Mode) @ 0x13f0e1a4 in /usr/bin/clickhouse
E           6. DB::BackupsWorker::doRestore(std::__1::shared_ptr<DB::ASTBackupQuery> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, DB::RestoreSettings, DB::BackupInfo const&, std::__1::shared_ptr<DB::IRestoreCoordination>, std::__1::shared_ptr<DB::Context>, bool) @ 0x13ec65c5 in /usr/bin/clickhouse
E           7. ? @ 0x13ec9cd9 in /usr/bin/clickhouse
E           8. ThreadPoolImpl<ThreadFromGlobalPool>::worker(std::__1::__list_iterator<ThreadFromGlobalPool, void*>) @ 0xa4c8606 in /usr/bin/clickhouse
E           9. void std::__1::__function::__policy_invoker<void ()>::__call_impl<std::__1::__function::__default_alloc_func<ThreadFromGlobalPool::ThreadFromGlobalPool<void ThreadPoolImpl<ThreadFromGlobalPool>::scheduleImpl<void>(std::__1::function<void ()>, int, std::__1::optional<unsigned long>)::'lambda0'()>(void&&)::'lambda'(), void ()> >(std::__1::__function::__policy_storage const*) @ 0xa4c9f57 in /usr/bin/clickhouse
E           10. ThreadPoolImpl<std::__1::thread>::worker(std::__1::__list_iterator<std::__1::thread, void*>) @ 0xa4c5f28 in /usr/bin/clickhouse
E           11. ? @ 0xa4c913d in /usr/bin/clickhouse
E           12. ? @ 0x7f8d21d3a609 in ?
E           13. __clone @ 0x7f8d21c5f133 in ?
E           . Stack trace:
E           
E           0. DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0xa40495a in /usr/bin/clickhouse
E           1. DB::readException(DB::ReadBuffer&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, bool) @ 0xa475bbe in /usr/bin/clickhouse
E           2. DB::BackupCoordinationStageSync::readCurrentState(std::__1::shared_ptr<zkutil::ZooKeeper>, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) const @ 0x13ee0017 in /usr/bin/clickhouse
E           3. DB::BackupCoordinationStageSync::waitImpl(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::optional<std::__1::chrono::duration<long long, std::__1::ratio<1l, 1000l> > >) const @ 0x13ede690 in /usr/bin/clickhouse
E           4. DB::RestoreCoordinationRemote::waitForStage(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) @ 0x13f06673 in /usr/bin/clickhouse
E           5. DB::BackupsWorker::doRestore(std::__1::shared_ptr<DB::ASTBackupQuery> const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, DB::RestoreSettings, DB::BackupInfo const&, std::__1::shared_ptr<DB::IRestoreCoordination>, std::__1::shared_ptr<DB::Context>, bool) @ 0x13ec687d in /usr/bin/clickhouse
E           6. DB::BackupsWorker::startRestoring(std::__1::shared_ptr<DB::IAST> const&, std::__1::shared_ptr<DB::Context>) @ 0x13ec0947 in /usr/bin/clickhouse
E           7. DB::BackupsWorker::start(std::__1::shared_ptr<DB::IAST> const&, std::__1::shared_ptr<DB::Context>) @ 0x13ebf093 in /usr/bin/clickhouse
E           8. DB::InterpreterBackupQuery::execute() @ 0x14e50f0d in /usr/bin/clickhouse
E           9. ? @ 0x151ba729 in /usr/bin/clickhouse
E           10. DB::executeQuery(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::shared_ptr<DB::Context>, bool, DB::QueryProcessingStage::Enum) @ 0x151b7bb7 in /usr/bin/clickhouse
E           11. DB::TCPHandler::runImpl() @ 0x15cb6cec in /usr/bin/clickhouse
E           12. DB::TCPHandler::run() @ 0x15ccb2f9 in /usr/bin/clickhouse
E           13. Poco::Net::TCPServerConnection::start() @ 0x18a41113 in /usr/bin/clickhouse
E           14. Poco::Net::TCPServerDispatcher::run() @ 0x18a4246d in /usr/bin/clickhouse
E           15. Poco::PooledThread::run() @ 0x18c04ea9 in /usr/bin/clickhouse
E           16. Poco::ThreadImpl::runnableEntry(void*) @ 0x18c02762 in /usr/bin/clickhouse
E           17. ? @ 0x7f8d21d3a609 in ?
E           18. __clone @ 0x7f8d21c5f133 in ?
E           . (CANNOT_RESTORE_TABLE)
E           (query: RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM Disk('backups', '22') SETTINGS replica_num_in_backup=1)

helpers/client.py:210: QueryRuntimeException
------------------------------ Captured log call -------------------------------
2023-08-03 01:16:43 [ 525 ] DEBUG : Executing query CREATE DATABASE mydb ON CLUSTER 'cluster' ENGINE=Replicated('/clickhouse/path/','{shard}','{replica}') on node1 (cluster.py:3054, query)
2023-08-03 01:16:44 [ 525 ] DEBUG : Executing query CREATE TABLE mydb.tbl (x UInt8, y String) ENGINE=ReplicatedMergeTree ORDER BY tuple() on node1 (cluster.py:3054, query)
2023-08-03 01:16:44 [ 525 ] DEBUG : Executing query ALTER TABLE mydb.tbl MODIFY COLUMN x String on node1 (cluster.py:3054, query)
2023-08-03 01:16:44 [ 525 ] DEBUG : Executing query BACKUP DATABASE mydb ON CLUSTER 'cluster' TO Disk('backups', '22') on node2 (cluster.py:3054, query)
2023-08-03 01:16:45 [ 525 ] DEBUG : Executing query DROP DATABASE mydb ON CLUSTER 'cluster' NO DELAY on node1 (cluster.py:3054, query)
2023-08-03 01:16:45 [ 525 ] DEBUG : Executing query RESTORE DATABASE mydb ON CLUSTER 'cluster' FROM Disk('backups', '22') SETTINGS replica_num_in_backup=1 on node1 (cluster.py:3054, query)
---------------------------- Captured log teardown -----------------------------
2023-08-03 01:16:46 [ 525 ] DEBUG : Executing query DROP TABLE IF EXISTS tbl ON CLUSTER 'cluster3' NO DELAY on node1 (cluster.py:3054, query)
2023-08-03 01:16:47 [ 525 ] DEBUG : Executing query DROP TABLE IF EXISTS tbl2 ON CLUSTER 'cluster3' NO DELAY on node1 (cluster.py:3054, query)
2023-08-03 01:16:47 [ 525 ] DEBUG : Executing query DROP DATABASE IF EXISTS mydb ON CLUSTER 'cluster3' NO DELAY on node1 (cluster.py:3054, query)
2023-08-03 01:16:47 [ 525 ] DEBUG : Executing query DROP USER IF EXISTS u1, u2 ON CLUSTER 'cluster3' on node1 (cluster.py:3054, query)

______________________________ test_store_cleanup ______________________________
[gw0] linux -- Python 3.8.10 /usr/bin/python3

started_cluster = <helpers.cluster.ClickHouseCluster object at 0x7fcbb50aec10>

    def test_store_cleanup(started_cluster):
        node1.query("CREATE DATABASE db UUID '10000000-1000-4000-8000-000000000001'")
        node1.query(
            "CREATE TABLE db.log UUID '10000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1"
        )
        node1.query(
            "CREATE TABLE db.mt UUID '10000000-1000-4000-8000-000000000003' ENGINE=MergeTree ORDER BY tuple() AS SELECT 1"
        )
        node1.query(
            "CREATE TABLE db.mem UUID '10000000-1000-4000-8000-000000000004' ENGINE=Memory AS SELECT 1"
        )

        node1.query("CREATE DATABASE db2 UUID '20000000-1000-4000-8000-000000000001'")
        node1.query(
            "CREATE TABLE db2.log UUID '20000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1"
        )
        node1.query("DETACH DATABASE db2")

        node1.query("CREATE DATABASE db3 UUID '30000000-1000-4000-8000-000000000001'")
        node1.query(
            "CREATE TABLE db3.log UUID '30000000-1000-4000-8000-000000000002' ENGINE=Log AS SELECT 1"
        )
        node1.query(
            "CREATE TABLE db3.log2 UUID '30000000-1000-4000-8000-000000000003' ENGINE=Log AS SELECT 1"
        )
        node1.query("DETACH TABLE db3.log")
        node1.query("DETACH TABLE db3.log2 PERMANENTLY")

        assert "d---------" not in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store"]
        )
        assert "d---------" not in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/100"]
        )
        assert "d---------" not in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/200"]
        )
        assert "d---------" not in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/300"]
        )

        node1.stop_clickhouse(kill=True)
        # All dirs related to `db` will be removed
        node1.exec_in_container(["rm", f"{path_to_data}/metadata/db.sql"])

        node1.exec_in_container(["mkdir", f"{path_to_data}/store/kek"])
        node1.exec_in_container(["touch", f"{path_to_data}/store/12"])
        try:
            node1.exec_in_container(["mkdir", f"{path_to_data}/store/456"])
        except Exception as e:
            print("Failed to create 456/:", str(e))
        node1.exec_in_container(["mkdir", f"{path_to_data}/store/456/testgarbage"])
        node1.exec_in_container(
            ["mkdir", f"{path_to_data}/store/456/30000000-1000-4000-8000-000000000003"]
        )
        node1.exec_in_container(
            ["touch", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000003"]
        )
        node1.exec_in_container(
            ["mkdir", f"{path_to_data}/store/456/45600000-1000-4000-8000-000000000004"]
        )

        node1.start_clickhouse()
        node1.query("DETACH DATABASE db2")
        node1.query("DETACH TABLE db3.log")

        node1.wait_for_log_line(
            "Removing access rights for unused directory",
            timeout=60,
            look_behind_lines=1000,
        )
        node1.wait_for_log_line("directories from store", look_behind_lines=1000)

        store = node1.exec_in_container(["ls", f"{path_to_data}/store"])
        assert "100" in store
        assert "200" in store
        assert "300" in store
        assert "456" in store
        assert "kek" in store
        assert "12" in store
        assert "d---------" in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store"]
        )
        assert "d---------" in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/456"]
        )

        # Metadata is removed, so store/100 contains garbage
        store100 = node1.exec_in_container(["ls", f"{path_to_data}/store/100"])
        assert "10000000-1000-4000-8000-000000000001" in store100
        assert "10000000-1000-4000-8000-000000000002" in store100
        assert "10000000-1000-4000-8000-000000000003" in store100
        assert "d---------" in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/100"]
        )

        # Database is detached, nothing to clean up
        store200 = node1.exec_in_container(["ls", f"{path_to_data}/store/200"])
        assert "20000000-1000-4000-8000-000000000001" in store200
        assert "20000000-1000-4000-8000-000000000002" in store200
        assert "d---------" not in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/200"]
        )

        # Tables are detached, nothing to clean up
        store300 = node1.exec_in_container(["ls", f"{path_to_data}/store/300"])
        assert "30000000-1000-4000-8000-000000000001" in store300
        assert "30000000-1000-4000-8000-000000000002" in store300
        assert "30000000-1000-4000-8000-000000000003" in store300
        assert "d---------" not in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/300"]
        )

        # Manually created garbage
        store456 = node1.exec_in_container(["ls", f"{path_to_data}/store/456"])
        assert "30000000-1000-4000-8000-000000000003" in store456
        assert "45600000-1000-4000-8000-000000000003" in store456
        assert "45600000-1000-4000-8000-000000000004" in store456
        assert "testgarbage" in store456
        assert "----------" in node1.exec_in_container(
            ["ls", "-l", f"{path_to_data}/store/456"]
        )

        node1.wait_for_log_line(
            "Removing unused directory", timeout=90, look_behind_lines=1000
        )
        node1.wait_for_log_line(
            "directories from store", timeout=90, look_behind_lines=1000000
        )

>       node1.wait_for_log_line(
            "Nothing to clean up from store/", timeout=90, look_behind_lines=1000000
        )

test_broken_detached_part_clean_up/test.py:286: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
helpers/cluster.py:3482: in wait_for_log_line
    result = self.exec_in_container(
helpers/cluster.py:3407: in exec_in_container
    return self.cluster.exec_in_container(
helpers/cluster.py:1751: in exec_in_container
    result = subprocess_check_call(
helpers/cluster.py:152: in subprocess_check_call
    return run_and_check(args, detach=detach, nothrow=nothrow)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

args = ['docker', 'exec', 'roottestbrokendetachedpartcleanup_node1_1', 'bash', '-c', 'timeout 90 tail -Fn1000000 "/var/log/clickhouse-server/clickhouse-server.log" | grep -Em 1 \'Nothing to clean up from store/\'']
env = None, shell = False, stdout = -1, stderr = -1, timeout = 300
nothrow = False, detach = False

    def run_and_check(
        args,
        env=None,
        shell=False,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        timeout=300,
        nothrow=False,
        detach=False,
    ):
        if detach:
            subprocess.Popen(
                args,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
                env=env,
                shell=shell,
            )
            return

        logging.debug(f"Command:{args}")
        res = subprocess.run(
            args, stdout=stdout, stderr=stderr, env=env, shell=shell, timeout=timeout
        )
        out = res.stdout.decode("utf-8")
        err = res.stderr.decode("utf-8")
        # check_call(...) from subprocess does not print stderr, so we do it manually
        for outline in out.splitlines():
            logging.debug(f"Stdout:{outline}")
        for errline in err.splitlines():
            logging.debug(f"Stderr:{errline}")
        if res.returncode != 0:
            logging.debug(f"Exitcode:{res.returncode}")
            if env:
                logging.debug(f"Env:{env}")
            if not nothrow:
>               raise Exception(
                    f"Command {args} return non-zero code {res.returncode}: {res.stderr.decode('utf-8')}"
                )
E               Exception: Command ['docker', 'exec', 'roottestbrokendetachedpartcleanup_node1_1', 'bash', '-c', 'timeout 90 tail -Fn1000000 "/var/log/clickhouse-server/clickhouse-server.log" | grep -Em 1 \'Nothing to clean up from store/\''] return non-zero code 1:

helpers/cluster.py:112: Exception