def get_taxids_by_scientific_name_wildcard(self, scientific_name):
        """
        Return all available taxid that fit the scientific name

        @attention: Several taxid might be a hit for one scientific name

        @param scientific_name: ncbi scientific name or synonym
        @type scientific_name: str

        @return: set of ncbi taxonomic identifiers
        @rtype: set[str | unicode] | None
        """
        assert isinstance(scientific_name, str)
        scientific_name = scientific_name.lower()
        matches = fnmatch.filter(self.name_to_taxids.keys(), scientific_name)
        set_of_tax_id = set()
        for match in matches:
            set_of_tax_id.update(set(self.name_to_taxids[match]))
        if len(set_of_tax_id) > 1:
            self._logger.warning(
                "Several matches '{}' found for scientific_name: '{}'".format(", ".join(matches), scientific_name))
            return set_of_tax_id
        elif len(set_of_tax_id) == 0:
            return None
        return set_of_tax_id 

def _apply_fname_filter(fnames, include, exclude):
    # todo: multi filter with *.csv|*.xls*|*.txt, split on |
    def helper(list_,filter_):
        return list(itertools.chain.from_iterable(fnmatch.filter(list_, ifilter) for ifilter in filter_.split('|')))
    if include:
        fnames = helper(fnames, include)

    if exclude:
        fnamesex = helper(fnames, exclude)
        fnames = list(set(fnames) - set(fnamesex))

    return fnames


#************************************************
# Pipe
#************************************************ 

def read_logs(self):
        self.list_logs = []
        #
        print "reading and parsing all the log files ... "
        #
        for path, dirs, files in os.walk(self.path_tracks):
            for name_file in fnmatch.filter(files, 'log.txt'):
                path_log = os.path.abspath(
                    os.path.join(path, name_file)
                )
                #
                self.list_logs.append(
                    self.parse_log(path_log)
                )
                #
            #
        #
        print "done ! "
        #
    #
    # 

def find_results(location, pattern):
    """ Create list of result files and return sorted

    Args:
      location (str): directory location to search
      pattern (str): glob style search pattern for results

    Returns:
      results (list): list of file paths for results found

    """
    # Note: already checked for location existence in main()
    records = []
    for root, dirnames, filenames in walk(location):
        for filename in fnmatch.filter(filenames, pattern):
            records.append(os.path.join(root, filename))

    if len(records) == 0:
        raise IOError('Could not find results in: %s' % location)

    records.sort()

    return records 

def idl2json(self):
        """
        Transform all IDL files in input folder to AVRO schemas in the output folder
        :return:
        """
        parser = argparse.ArgumentParser(
            description='Converts IDL to Avro JSON schema')
        # prefixing the argument with -- means it's optional
        parser.add_argument('--input', help='Input folder containing *.avdl files')
        parser.add_argument('--output', help='Output folder for the JSON schemas')
        args = parser.parse_args(sys.argv[2:])
        logging.info('idl2schema')
        makedir(args.output)
        idls = [os.path.join(dirpath, f)
                for dirpath, dirnames, files in os.walk(args.input)
                for f in fnmatch.filter(files, "*.{}".format(IDL_EXTENSION))]

        for idl in idls:
            logging.info("Transforming: " + idl)
            idl2schemata_command = "java -jar {} idl2schemata {} {}".format(
                AVRO_TOOLS_JAR, idl, args.output
            )
            logging.info("Running: [%s]" % idl2schemata_command)
            run_command(idl2schemata_command) 

def idl2avpr(self):
        """
        Transform all IDL files in input folder to AVPRs in the output folder
        :return:
        """
        parser = argparse.ArgumentParser(
            description='Converts IDL to AVPR')
        # prefixing the argument with -- means it's optional
        parser.add_argument('--input', help='Input folder containing *.avdl files')
        parser.add_argument('--output', help='Output folder for the AVPRs')
        args = parser.parse_args(sys.argv[2:])
        logging.info('idl2avpr')
        makedir(args.output)
        idls = [os.path.join(dirpath, f)
                for dirpath, dirnames, files in os.walk(args.input)
                for f in fnmatch.filter(files, "*.{}".format(IDL_EXTENSION))]
        for idl in idls:
            logging.info("Transforming: " + idl)
            file_name = os.path.basename(idl).replace(IDL_EXTENSION, AVPR_EXTENSION)
            idl2avpr_command = "java -jar {} idl {} {}/{}".format(
                AVRO_TOOLS_JAR, idl, args.output, file_name
            )
            logging.info("Running: [%s]" % idl2avpr_command)
            run_command(idl2avpr_command) 

def json2java(self):
        """
        Transform all JSON Avro schemas in a given folder to Java source code.
        :return:
        """
        parser = argparse.ArgumentParser(
            description='Generates Java source code from Avro schemas')
        # NOT prefixing the argument with -- means it's not optional
        parser.add_argument('--input', help='Input folder containing *.avdl files')
        parser.add_argument('--output', help='Output folder for the Java source code')
        args = parser.parse_args(sys.argv[2:])
        logging.info('json2java')
        makedir(args.output)
        jsons = [os.path.join(dirpath, f)
                for dirpath, dirnames, files in os.walk(args.input)
                for f in fnmatch.filter(files, "*.{}".format(JSON_EXTENSION))]
        for json in jsons:
            logging.info("Transforming: " + json)
            idl2avpr_command = "java -jar {} compile -string schema {} {}".format(
                AVRO_TOOLS_JAR, json, args.output
            )
            logging.info("Running: [%s]" % idl2avpr_command)
            run_command(idl2avpr_command) 

def exclude_data_files(self, package, src_dir, files):
        """Filter filenames for package's data files in 'src_dir'"""
        globs = (
            self.exclude_package_data.get('', [])
            + self.exclude_package_data.get(package, [])
        )
        bad = set(
            item
            for pattern in globs
            for item in fnmatch.filter(
                files,
                os.path.join(src_dir, convert_path(pattern)),
            )
        )
        seen = collections.defaultdict(itertools.count)
        return [
            fn
            for fn in files
            if fn not in bad
            # ditch dupes
            and not next(seen[fn])
        ] 

def update(path, data, dryrun=False, remote=False, gradle=False, interact=False):
    global is_dryrun
    is_dryrun = dryrun
    global check_remote
    check_remote = remote
    global check_gradle
    check_gradle = gradle
    global is_interact
    is_interact = interact

    for file in filter(path, "build.gradle"):
        parse_dependency(file, data)

    if check_gradle:
        gradle_version = andle.gradle.load()
        for file in filter(path, "gradle-wrapper.properties"):
            parse_gradle(file, gradle_version) 

def get_images_count_recursive(path):
    matches = []
    score_iou = []
    # logging.debug('path {}'.format(path))
    for root, dirnames, filenames in sorted(os.walk(path)):
        for filename in sorted(fnmatch.filter(filenames, '*.jpg')):
            # logging.debug('filename {}'.format(filename))
            matches.append(os.path.join(root, filename))
            score_iou.append(filename.split('_')[-1].replace('.jpg',''))

    # logging.debug('matches {}'.format(matches))

    images_count = len(matches)
    return score_iou, images_count

# nb_train_samples = 2000
# nb_validation_samples = 800 

def find_data_files(self, package, src_dir):
        """Return filenames for package's data files in 'src_dir'"""
        patterns = self._get_platform_patterns(
            self.package_data,
            package,
            src_dir,
        )
        globs_expanded = map(glob, patterns)
        # flatten the expanded globs into an iterable of matches
        globs_matches = itertools.chain.from_iterable(globs_expanded)
        glob_files = filter(os.path.isfile, globs_matches)
        files = itertools.chain(
            self.manifest_files.get(package, []),
            glob_files,
        )
        return self.exclude_data_files(package, src_dir, files) 

def exclude_data_files(self, package, src_dir, files):
        """Filter filenames for package's data files in 'src_dir'"""
        files = list(files)
        patterns = self._get_platform_patterns(
            self.exclude_package_data,
            package,
            src_dir,
        )
        match_groups = (
            fnmatch.filter(files, pattern)
            for pattern in patterns
        )
        # flatten the groups of matches into an iterable of matches
        matches = itertools.chain.from_iterable(match_groups)
        bad = set(matches)
        keepers = (
            fn
            for fn in files
            if fn not in bad
        )
        # ditch dupes
        return list(_unique_everseen(keepers)) 

def find_data_files(self, package, src_dir):
        """Return filenames for package's data files in 'src_dir'"""
        patterns = self._get_platform_patterns(
            self.package_data,
            package,
            src_dir,
        )
        globs_expanded = map(glob, patterns)
        # flatten the expanded globs into an iterable of matches
        globs_matches = itertools.chain.from_iterable(globs_expanded)
        glob_files = filter(os.path.isfile, globs_matches)
        files = itertools.chain(
            self.manifest_files.get(package, []),
            glob_files,
        )
        return self.exclude_data_files(package, src_dir, files) 

def walk(self, func, arg):
        """
        Walk this directory tree by calling the specified function
        for each directory in the tree.

        This behaves like the os.path.walk() function, but for in-memory
        Node.FS.Dir objects.  The function takes the same arguments as
        the functions passed to os.path.walk():

                func(arg, dirname, fnames)

        Except that "dirname" will actually be the directory *Node*,
        not the string.  The '.' and '..' entries are excluded from
        fnames.  The fnames list may be modified in-place to filter the
        subdirectories visited or otherwise impose a specific order.
        The "arg" argument is always passed to func() and may be used
        in any way (or ignored, passing None is common).
        """
        entries = self.entries
        names = list(entries.keys())
        names.remove('.')
        names.remove('..')
        func(arg, self, names)
        for dirname in [n for n in names if isinstance(entries[n], Dir)]:
            entries[dirname].walk(func, arg) 

def exclude_data_files(self, package, src_dir, files):
        """Filter filenames for package's data files in 'src_dir'"""
        globs = (self.exclude_package_data.get('', [])
                 + self.exclude_package_data.get(package, []))
        bad = []
        for pattern in globs:
            bad.extend(
                fnmatch.filter(
                    files, os.path.join(src_dir, convert_path(pattern))
                )
            )
        bad = dict.fromkeys(bad)
        seen = {}
        return [
            f for f in files if f not in bad
                and f not in seen and seen.setdefault(f,1)  # ditch dupes
        ] 

def find_data_files(self, package, src_dir):
        """Return filenames for package's data files in 'src_dir'"""
        patterns = self._get_platform_patterns(
            self.package_data,
            package,
            src_dir,
        )
        globs_expanded = map(glob, patterns)
        # flatten the expanded globs into an iterable of matches
        globs_matches = itertools.chain.from_iterable(globs_expanded)
        glob_files = filter(os.path.isfile, globs_matches)
        files = itertools.chain(
            self.manifest_files.get(package, []),
            glob_files,
        )
        return self.exclude_data_files(package, src_dir, files) 

def exclude_data_files(self, package, src_dir, files):
        """Filter filenames for package's data files in 'src_dir'"""
        files = list(files)
        patterns = self._get_platform_patterns(
            self.exclude_package_data,
            package,
            src_dir,
        )
        match_groups = (
            fnmatch.filter(files, pattern)
            for pattern in patterns
        )
        # flatten the groups of matches into an iterable of matches
        matches = itertools.chain.from_iterable(match_groups)
        bad = set(matches)
        keepers = (
            fn
            for fn in files
            if fn not in bad
        )
        # ditch dupes
        return list(_unique_everseen(keepers)) 

def pytest_generate_tests(metafunc):
    if 'scenario' in metafunc.fixturenames:
        scenarios = {}
        for root, dirnames, filenames in os.walk('tests/scenarios'):
            for filename in fnmatch.filter(filenames, '*.yml'):
                filepath = os.path.join(root, filename)
                scenarios.update(yaml.load(io.open(filepath, encoding='utf-8')) or {})
        params = []
        for name in sorted(scenarios):
            params.append([name, scenarios[name]])
        metafunc.parametrize('name, scenario', params) 

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename) 

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename) 

def glob_recursive(path, pattern):
    matches = []
    for dirpath, dirnames, filenames in os.walk(path):
        for filename in fnmatch.filter(filenames, pattern):
            matches.append(os.path.join(dirpath, filename))
    return matches 

def clean(cls):
        if os.path.exists(cls.output_path):
            for dirpath, dirnames, filenames in os.walk(cls.output_path):
                for filename in fnmatch.filter(filenames, '*.ptx'):
                    os.remove(os.path.join(dirpath, filename)) 

def glob_recursive(path, pattern):
    matches = []
    for dirpath, dirnames, filenames in os.walk(path):
        for filename in fnmatch.filter(filenames, pattern):
            matches.append(os.path.join(dirpath, filename))
    return matches 

def globber_recursive(root, patt):
    for root, dirnames, filenames in walk(root):
        for filename in fnmatch.filter(filenames, patt):
            yield join(root, filename) 

def ignore_patterns(*patterns):
    """Function that can be used as copytree() ignore parameter.

    Patterns is a sequence of glob-style patterns
    that are used to exclude files"""
    def _ignore_patterns(path, names):
        ignored_names = []
        for pattern in patterns:
            ignored_names.extend(fnmatch.filter(names, pattern))
        return set(ignored_names)
    return _ignore_patterns 

def recursive_glob(root, pattern):
    if root is None:
        return

    if os.path.isfile(root) and fnmatch.fnmatch(root, pattern):
        yield root
        return

    for base, dirs, files in os.walk(root):
        matches = fnmatch.filter(files, pattern)
        for filename in matches:
            yield os.path.join(base, filename) 

def find_jars_within_path(path_to_jars):
	return [os.path.join(root, filename) 
			for root, dirnames, filenames in os.walk(path_to_jars) 
			for filename in fnmatch.filter(filenames, '*.jar')] 

def find_files(pattern, root):
  """Return all the files matching pattern below root dir."""
  for dirpath, _, files in os.walk(root):
    for filename in fnmatch.filter(files, pattern):
      yield os.path.join(dirpath, filename) 

def matchGtWithPreds(predictionList,groundTruthList,gtInstances,args):
    matches = {}
    if not args.quiet:
        print("Matching {} pairs of images...".format(len(predictionList)))

    count = 0
    for (pred,gt) in zip(predictionList,groundTruthList):
        # key for dicts
        dictKey = os.path.abspath(gt)

        # Read input files
        gtImage  = readGTImage(gt,args)
        predInfo = readPredInfo(pred,args)

        # Get and filter ground truth instances
        unfilteredInstances = gtInstances[ dictKey ]
        curGtInstancesOrig  = filterGtInstances(unfilteredInstances,args)

        # Try to assign all predictions
        (curGtInstances,curPredInstances) = assignGt2Preds(curGtInstancesOrig, gtImage, predInfo, args)

        # append to global dict
        matches[ dictKey ] = {}
        matches[ dictKey ]["groundTruth"] = curGtInstances
        matches[ dictKey ]["prediction"]  = curPredInstances

        count += 1
        if not args.quiet:
            print("\rImages Processed: {}".format(count), end=' ')
            sys.stdout.flush()

    if not args.quiet:
        print("")

    return matches

# For a given frame, assign all predicted instances to ground truth instances 

def clear(self):
        # imported lazily here because google app-engine doesn't support
        # write access on the file system and the function does not exist
        # normally.
        from os import remove
        files = fnmatch.filter(listdir(self.directory), self.pattern % '*')
        for filename in files:
            try:
                remove(path.join(self.directory, filename))
            except OSError:
                pass