def __init__(self, docker_config):
        self.log_level = os.getenv('PYWREN_LOGLEVEL')
        self.config = docker_config
        self.name = 'docker'
        self.host = docker_config['host']
        self.queue = multiprocessing.Queue()
        self.docker_client = None
        self._is_localhost = self.host in ['127.0.0.1', 'localhost']

        if self._is_localhost:
            try:
                self.docker_client = docker.from_env()
            except Exception:
                pass

        log_msg = 'PyWren v{} init for Docker - Host: {}'.format(__version__, self.host)
        logger.info(log_msg)
        if not self.log_level:
            print(log_msg) 

def __init__(self):
        """ Connnects to the docker daemon"""
        # will be used as the tag on the docker image
        self.problem_name = sanitize_name(self.name)
        # use an explicit remote docker daemon per the configuration
        try:
            tls_config = docker.tls.TLSConfig(
                ca_cert=self.docker_ca_cert,
                client_cert=(self.docker_client_cert, self.docker_client_key),
                verify=True)

            self.client = docker.DockerClient(base_url=self.docker_host, tls=tls_config)
            self.api_client = docker.APIClient(base_url=self.docker_host, tls=tls_config)
            logger.debug("Connecting to docker daemon with config")

        # Docker options not set in configuration so use the environment to
        # configure (could be local or remote)
        except AttributeError:
            logger.debug("Connecting to docker daemon with env")
            self.client = docker.from_env()

        # throws an exception if the server returns an error: docker.errors.APIError
        self.client.ping() 

def _start_bitcoind(self, cleanup_at_exit):
        bitcoind_path = self.construct_bitcoind_cmd(self.rpcconn )
        dclient = docker.from_env()
        logger.debug("Running (in docker): {}".format(bitcoind_path))
        ports={
            '{}/tcp'.format(self.rpcconn.rpcport-1): self.rpcconn.rpcport-1, 
            '{}/tcp'.format(self.rpcconn.rpcport): self.rpcconn.rpcport
        }
        logger.debug("portmapping: {}".format(ports))
        image = dclient.images.get("registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:{}".format(self.docker_tag))
        self.btcd_container = dclient.containers.run("registry.gitlab.com/cryptoadvance/specter-desktop/python-bitcoind:{}".format(self.docker_tag), bitcoind_path,  ports=ports, detach=True)
        def cleanup_docker_bitcoind():
            self.btcd_container.stop()
            self.btcd_container.remove()
        if cleanup_at_exit:
            atexit.register(cleanup_docker_bitcoind)
        logger.debug("Waiting for container {} to come up".format(self.btcd_container.id))
        self.wait_for_container()
        rpcconn, _ = self.detect_bitcoind_container(self.rpcconn.rpcport)
        if rpcconn == None:
            raise Exception("Couldn't find container or it died already. Check the logs!")
        else:
            self.rpcconn = rpcconn
        return 

def test_creates_service(hub_service):
    """Test that logging in as a new user creates a new docker service."""
    client = docker.from_env()

    services_before_login = client.services.list()

    # This request should create a new docker service to run the server for a-new-user
    response = requests.post("http://127.0.0.1:8000/hub/login?next=", data={"username": "a-new-user", "password": "just magnets"})

    assert response.status_code == 200

    services_after_login = client.services.list()
    assert len(services_after_login) - len(services_before_login) == 1

    # Remove the service we just created, or we'll get errors when tearing down the fixtures
    (set(services_after_login) - set(services_before_login)).pop().remove() 

def __init__(self, toxinidir, distdir):
        """
        :param toxinidir: directory containing tox.ini
        :type toxinidir: str
        :param distdir: tox dist directory
        :type distdir: str
        """
        self._toxinidir = toxinidir
        self._distdir = distdir
        self._gitdir = os.path.join(self._toxinidir, '.git')
        logger.debug('Initializing DockerImageBuilder; toxinidir=%s gitdir=%s '
                     'distdir=%s',
                     self._toxinidir, self._gitdir, self._distdir)
        if not os.path.exists(self._gitdir) or not os.path.isdir(self._gitdir):
            raise RuntimeError(
                'Error: %s does not exist or is not a directory' % self._gitdir
            )
        logger.debug('Connecting to Docker')
        self._docker = docker.from_env() 

def docker_memory_limit():
    docker_client = docker.from_env()
    # Get memory limit from docker container through the API
    # Because the docker execution may be remote

    cmd = f'python3 -u -c "import os; print(int(os.sysconf("SC_PAGE_SIZE") '\
          f'* os.sysconf("SC_PHYS_PAGES") / (1024. ** 2)) // {CELERY_WORKER_CONCURRENCY}), flush=True, end=\'\')"'

    task_args = {
        'image': CELERYWORKER_IMAGE,
        'command': cmd,
        'detach': False,
        'stdout': True,
        'stderr': True,
        'auto_remove': False,
        'remove': True,
        'network_disabled': True,
        'network_mode': 'none',
        'privileged': False,
        'cap_drop': ['ALL'],
    }

    memory_limit_bytes = docker_client.containers.run(**task_args)

    return int(memory_limit_bytes) 

def remove_local_folders(compute_plan_id):
    if not settings.ENABLE_REMOVE_LOCAL_CP_FOLDERS:
        logger.info(f'Skipping remove local volume of compute plan {compute_plan_id}')
        return

    logger.info(f'Remove local volume of compute plan {compute_plan_id}')

    client = docker.from_env()
    volume_id = get_volume_id(compute_plan_id)

    try:
        local_volume = client.volumes.get(volume_id=volume_id)
        local_volume.remove(force=True)
    except docker.errors.NotFound:
        pass
    except Exception:
        logger.error(f'Cannot remove volume {volume_id}', exc_info=True)

    if settings.TASK['CHAINKEYS_ENABLED']:
        chainkeys_directory = get_chainkeys_directory(compute_plan_id)
        try:
            shutil.rmtree(chainkeys_directory)
        except Exception:
            logger.error(f'Cannot remove volume {chainkeys_directory}', exc_info=True) 

def test_exception_handler(self):

        # Python exception in system
        try:
            1 / 0
        except Exception as e:
            error_code = compute_error_code(e)
            value_error_code, _ = get_exception_code(ZeroDivisionError)
            self.assertIn(f'00-01-{value_error_code}', error_code)

        # Python exception in docker
        try:
            client = docker.from_env()
            client.containers.run("python:3.6", ['python3', '-c', 'print(KO)'], remove=True)
        except Exception as e:
            error_code = compute_error_code(e)
            container_error_code, _ = get_exception_code(NameError)
            self.assertIn(f'01-01-{container_error_code}', error_code) 

def removeServices(serviceid):
    logging.info('Remove the service %s', serviceid)
    
    try:
        docker_client = docker.from_env()
        docker_remove = docker_client.services.get(serviceid)
        docker_remove.remove()
        remove_ser = models.Service.query.all()
        for i in remove_ser:
            if (i.serviceid == serviceid):
                db.session.delete(i)
                db.session.commit()
                break
               
    except docker.errors.APIError as e:
        if e.status_code == 404:
            remove_ser = models.Service.query.all()
            for i in remove_ser:
                if (i.serviceid == serviceid):
                    db.session.delete(i)
                    db.session.commit()
                    break
        else:
            logging.error('Unable to remove the service %s. \nReason: %s', serviceid, str(e)) 

def deleteImage(image_name):
    logging.info('Delete the image %s', image_name)
    try:
        docker_client = docker.from_env()
        registry_imagename = registry + '/' + image_name
        docker_client.images.remove(image=registry_imagename,force=True)
        image = models.Image.query.filter_by(imagename=image_name).first()
        db.session.delete(image)
        db.session.commit()
    except docker.errors.APIError as e:
        image = models.Image.query.filter_by(imagename = image_name).first()
        db.session.delete(image)
        db.session.commit()
        error_string = 'Unable to delete the image {}. \nReason: {}. Delete the record'.format(registry_imagename, str(e))
        logging.error(error_string)
        return error_string
    
    return None 

def setUpClass(cls):
        print(' \n >>>> ==== Running Test 2 - Verify Mounted Volume')
        print(' >>>> ==== using IMAGE = ' + cls.docker_image + ' ===== ')
        DockerUtil.empty_test_folder("tmp")
        # Setup test dir as volume to by mounted by container
        appdata = DockerUtil.create_test_dir("tmp/appdata")
        exts =  DockerUtil.create_test_dir("tmp/extensions")
        mount={}
        mount[appdata]= { 'bind':'/opt/connect/appdata', 'mode':'rw'}
        mount[exts]= {'bind':'/opt/connect/custom-extensions','mode':'ro'}
        # run docker image with -v option
        client = docker.from_env()
        cls.container = client.containers.run(cls.docker_image,volumes=mount,detach=True,name="mctest2")
        # wait for MC to come up
        try:
            DockerUtil.wait_for_containers([cls.container], 60)
        except Exception, e:
            print(">>>> MC server failed to start")
            cls.tearDownClass()
            raise e 

def setUpClass(cls):
        # run docker image with 2 environment variables
        print(' \n >>>> ==== Running Test 1 - Verify Environment Variables')
        print(' >>>> ==== using IMAGE = ' + cls.docker_image + ' ===== ')
        client = docker.from_env()
        cls.container = client.containers.run(cls.docker_image, 
            environment=[
                "SESSION_STORE=true",
                "VMOPTIONS=-Xmx768m"
            ],
            detach=True, 
            name="mctest1")
        # wait for MC to come up
        try:
            DockerUtil.wait_for_containers([cls.container], 60)
        except Exception, e:
            print(">>>> MC server failed to start")
            cls.tearDownClass()
            raise e
        # retrieve container mirth.properties file as a map 

def setUpClass(cls):
        print(' \n >>>> ==== Running Test 3 - Verify Compose with secret, postgres, custom-extensions') 
        print( ' >>>> ==== using IMAGE = ' + cls.docker_image + ' ===== ')
        DockerUtil.empty_test_folder("tmp")
        # Setup test dir as volume to by mounted by container
        exts =  DockerUtil.create_test_dir("tmp/exts")
        os.system('cp ./testdata/*.zip ./tmp/exts/')
        os.system('cp ./testdata/secret.properties ./tmp/')
        DockerUtil.generate_compose_yml('./tmp/test.yml',cls.docker_image)
        # Run docker compose        
        os.system(cls.composeCmd + " up -d")
        client = docker.from_env()
        cls.container = client.containers.get("mctest3_mc_1")
        # wait for MC to come up
        try:
            DockerUtil.wait_for_containers([cls.container], 60)
        except Exception, e:
            print(">>>> MC server failed to start")
            cls.tearDownClass()
            raise e
        # retrieve container mirth.properties file as a map 

def test_local_docker_conf(self):
        """ Test to connect to a local Docker daemon """
        try:
            docker_connection = docker.from_env()
        except Exception as e:
            self._display_error("- Unable to connect to Docker. Error was %s" % str(e))
            return False

        try:
            self._display_info("- Asking Docker some info")
            if docker.utils.compare_version('1.24', docker_connection.version()['ApiVersion']) < 0:
                self._display_error("- Docker version >= 1.12.0 is required.")
                return False
        except Exception as e:
            self._display_error("- Unable to contact Docker. Error was %s" % str(e))
            return False
        self._display_info("- Successfully got info from Docker. Docker connection works.")

        return True 

def download_containers(self, to_download, current_options):
        """ Download the chosen containers on all the agents """
        if current_options["backend"] == "local":
            self._display_info("Connecting to the local Docker daemon...")
            try:
                docker_connection = docker.from_env()
            except:
                self._display_error("Cannot connect to local Docker daemon. Skipping download.")
                return

            for image in to_download:
                try:
                    self._display_info("Downloading image %s. This can take some time." % image)
                    docker_connection.images.pull(image + ":latest")
                except Exception as e:
                    self._display_error("An error occurred while pulling the image: %s." % str(e))
        else:
            self._display_warning(
                "This installation tool does not support the backend configuration directly, if it's not local. You will have to "
                "pull the images by yourself. Here is the list: %s" % str(to_download)) 

def setUp(self):
        self.parser = cli.cli.make_parser()

        self.docker_client = docker.from_env()
        smppSimulatorName = "nazTestSmppSimulator"
        running_containers = self.docker_client.containers.list()
        for container in running_containers:
            container.stop()

        self.smpp_server = self.docker_client.containers.run(
            "komuw/smpp_server:v0.3",
            name=smppSimulatorName,
            detach=True,
            auto_remove=True,
            labels={"name": "smpp_server", "use": "running_naz_tets"},
            ports={"2775/tcp": 2775, "8884/tcp": 8884},
            stdout=True,
            stderr=True,
        )
        self.naz_config = "tests.test_cli.NAZ_CLIENT"
        self.bad_naz_config = "tests.test_cli.BAD_NAZ_CLIENT" 

def setUpClass(cls):
        docker_client = docker.from_env()
        smppSimulatorName = "nazTestSmppSimulator"
        running_containers = docker_client.containers.list()
        for container in running_containers:
            container.kill()
        cls.smpp_server = docker_client.containers.run(
            "komuw/smpp_server:v0.3",
            name=smppSimulatorName,
            detach=True,
            auto_remove=True,
            labels={"name": "smpp_server", "use": "running_naz_tets"},
            ports={"{0}/tcp".format(TestClient.smsc_port): TestClient.smsc_port, "8884/tcp": 8884},
            stdout=True,
            stderr=True,
        )
        # sleep to give enough time for the smpp_server container to have started properly
        time.sleep(1.0) 

def get_profile_image(profile):
    try:
        global docker_api
        if not docker_api:
            docker_api = docker.from_env(version='auto')

        image_name = PROFILE_PREFIX + profile
        image = docker_api.images.get(image_name)
        assert image.labels.get(LABEL_BROWSERPROFILE) == profile
        return 'profile:' + profile

    except (docker.errors.ImageNotFound, AssertionError):
        if not is_quiet():
            print('Profile "{0}" not found'.format(profile))
        sys.exit(1)


# ============================================================================ 

def print_logs(browsers, follow=False, wait=False, all_containers=False):
    docker_api = docker.from_env(version='auto')

    if follow is None:
        follow = False

    for reqid in browsers:
        if all_containers:
            print_container_log(
                docker_api, reqid, wait=False, follow=False, name='browser-'
            )

            print_container_log(
                docker_api, reqid, wait=False, follow=False, name='xserver-'
            )

        print_container_log(docker_api, reqid, wait=wait, follow=follow)


# ============================================================================ 

def __init__(self):
        self.client = docker.from_env()

        try:
            with open('manifest.json', 'r') as os_manifest:
                self.OS_LIST = json.load(os_manifest)
        except Exception:
            pass

        if self.OS_LIST is None:
            raise Exception(
                'Could not load manifest.json. ' +
                'Download it from https://get.instantbox.org/manifest.json'
            )

        self.AVAILABLE_OS_LIST = []
        for os in self.OS_LIST:
            for ver in os['subList']:
                self.AVAILABLE_OS_LIST.append(ver['osCode']) 

def create_docker_client(docker_host: str = '', check=True) -> docker.DockerClient:
    """
    Context manager for DockerClient creation

    :param docker_host: DOCKER_HOST arg for DockerClient
    :param check: check if docker is available
    :return: DockerClient instance
    """
    with _docker_host_lock:
        os.environ["DOCKER_HOST"] = docker_host  # The env var DOCKER_HOST is used to configure docker.from_env()
        client = docker.from_env()
    if check and not _is_docker_running(client):
        raise RuntimeError("Docker daemon is unavailable")
    try:
        yield client
    finally:
        client.close() 

def client(self) -> 'docker.DockerClient':
        return docker.from_env() 

def client(self) -> 'docker.DockerClient':
        return docker.from_env() 

def __init__(self, job_backend, cpu_cores=1, gpu_devices=None, docker_container=None):
        Thread.__init__(self)

        self.job_backend = job_backend
        self.gpu_devices = gpu_devices
        self.docker_container = docker_container
        self.max_minutes = 0
        self.cpu_cores = cpu_cores

        job = self.job_backend.job
        if 'maxTime' in job['config'] and isinstance(job['config']['maxTime'], int) and job['config']['maxTime'] > 0:
            self.max_minutes = job['config']['maxTime']

        self.hardware_stream = self.job_backend.git.stream_file('aetros/job/monitoring.csv')

        header = ["second", "cpu", "memory"]
        try:
            if self.gpu_devices:
                for gpu_id, gpu in enumerate(aetros.cuda_gpu.get_ordered_devices()):
                    if gpu_id in gpu_devices:
                        header.append("memory_gpu" + str(gpu['id']))
        except aetros.cuda_gpu.CudaNotImplementedException: pass

        if job_backend.get_job_model().has_dpu():
            header += ['dpu0']

        self.hardware_stream.write(simplejson.dumps(header)[1:-1] + "\n")
        self.running = True
        self.early_stopped = False
        self.handle_max_time = True
        self.client = docker.from_env()
        self.docker_api = docker.APIClient(**docker.utils.kwargs_from_env())
        self.stat_stream = None

        self.docker_last_last_reponse = None
        self.docker_last_stream_data = 0
        self.docker_last_mem = None
        self.docker_last_cpu = None 

def search_bitcoind_container(all=False):
        ''' returns a list of containers which are running bitcoind '''
        d_client = docker.from_env()
        return [c for c in d_client.containers.list(all) if (c.attrs['Config'].get('Cmd')or[""])[0]=='bitcoind'] 

def detect_bitcoind_container(with_rpcport):
        ''' checks all the containers for a bitcoind one, parses the arguments and initializes 
            the object accordingly 
            returns rpcconn, btcd_container
        '''
        d_client = docker.from_env()
        potential_btcd_containers = BitcoindDockerController.search_bitcoind_container()
        if len(potential_btcd_containers) == 0:
            logger.debug("could not detect container. Candidates: {}".format(d_client.containers.list()))
            all_candidates = BitcoindDockerController.search_bitcoind_container(all=True)
            logger.debug("could not detect container. All Candidates: {}".format(all_candidates))
            if len(all_candidates) > 0:
                logger.debug("100 chars of logs of first candidate")
                logger.debug(all_candidates[0].logs()[0:100])
            return None
        for btcd_container in potential_btcd_containers:
            rpcport = int([arg for arg in btcd_container.attrs['Config']['Cmd'] if 'rpcport' in arg][0].split('=')[1])
            if rpcport != with_rpcport:
                logger.debug("checking port {} against searched port {}".format(type(rpcport), type(with_rpcport)))
                continue
            rpcpassword = [arg for arg in btcd_container.attrs['Config']['Cmd'] if 'rpcpassword' in arg][0].split('=')[1]
            rpcuser = [arg for arg in btcd_container.attrs['Config']['Cmd'] if 'rpcuser' in arg][0].split('=')[1]
            if "CI" in os.environ: # this is a predefined variable in gitlab
                # This works on Linux (direct docker) and gitlab-CI but not on MAC
                ipaddress = btcd_container.attrs['NetworkSettings']['IPAddress']
            else:
                # This works on most machines but not on gitlab-CI
                ipaddress ="127.0.0.1"
            rpcconn = Btcd_conn(rpcuser=rpcuser, rpcpassword=rpcpassword, rpcport=rpcport, ipaddress=ipaddress)
            logger.info("detected container {}".format(btcd_container.id))
            return rpcconn, btcd_container
        logger.debug("No matching container found")
        return None 

def _jupyterhub_docker_tags(self):
        try:
            include_jh_tags = lambda tag: self.jupyterhub_docker_tag_re.match(tag)
            return filter(include_jh_tags, [tag for image in docker.from_env().images.list() for tag in image.tags])
        except NameError:
            raise Exception('The docker package is not installed and is a dependency for DockerProfilesSpawner') 

def openshift_container(request, port, pytestconfig):
    capmanager = pytestconfig.pluginmanager.getplugin('capturemanager')
    client = docker.from_env()
    openshift_version = request.config.getoption('--openshift-version')
    if openshift_version is None:
        yield None
    else:
        container = client.containers.run(
            'openshift/origin:{}'.format(openshift_version),
            'start master --listen=0.0.0.0:{}'.format(port),
            detach=True,
            ports={port: port}
        )

        try:
            # Wait for the container to no longer be in the created state before
            # continuing
            while container.status == u'created':
                capmanager.suspend()
                print("\nWaiting for container...")
                capmanager.resume()
                time.sleep(5)
                container = client.containers.get(container.id)

            # Disable InsecureRequest warnings because we can't get the cert yet
            warnings.simplefilter('ignore', InsecureRequestWarning)
            # Wait for the api server to be ready before continuing
            for _ in range(10):
                try:
                    requests.head("https://127.0.0.1:{}/healthz/ready".format(port), verify=False)
                except requests.RequestException:
                    pass
                time.sleep(1)
            warnings.simplefilter('default', InsecureRequestWarning)

            time.sleep(1)

            yield container
        finally:
            # Always remove the container
            container.remove(force=True) 

def cleanup_hanged_docker_containers():
    try:
        client = docker.from_env()
        for container in client.containers.list():
            if container.name == "pytest_mysql_to_sqlite3":
                container.kill()
                break
    except Exception:
        pass 

def pytest_keyboard_interrupt():
    try:
        client = docker.from_env()
        for container in client.containers.list():
            if container.name == "pytest_mysql_to_sqlite3":
                container.kill()
                break
    except Exception:
        pass