Source code for pymlp.datahandling.MultifolderReader

import numpy as np
from pymlp.datahandling.DataHandler import DataHandler

import os
import re

[docs]class MultifolderReader: """Reads all datasets in all subfolders of the given folder. Filter the subfolders and files and sorts the datasets and folder names.""" def __init__(self, folder, folderfilter, filefilter, negativefilter=None): """ Parameters: folder -- the folder to look for data containing subfolders folderfilter -- folders that do not end with this filter are not analyzed filefilter -- needs to be in the files to read """ self.folder = folder self.folderfilter = folderfilter self.filefilter = filefilter self.negativefilter = negativefilter
[docs] def collectData(self): folders = os.listdir(self.folder) relativeFolderNames = map(lambda f: "%s/%s" % (self.folder, f), folders) relativeFolderNames = filter(lambda x: os.path.isdir(x), relativeFolderNames) foldersToRead = filter(lambda x: x.endswith(self.folderfilter), relativeFolderNames) if self.negativefilter != None: foldersToRead = filter(lambda x: not self.negativefilter in x, relativeFolderNames) foldersToRead.sort(self._folderComparator) data = map(self._collectRawData, foldersToRead) return data, foldersToRead
def _collectRawData(self, folder): analyzer = DataHandler(folder, steplength=1) files = analyzer.filterFiles(lambda f: self.filefilter in f) content = map(analyzer.loadData, files) content = np.array(content) # assert content.ndim == 2, "%s == 2" % content.ndim return content
[docs] def parseParameters(self, line): parameters = line.split("/")[-1] layoutPart = parameters layoutWithEmptyStrings = re.split("[\(, \)]", layoutPart) nonLayoutParameters= map(float, layoutWithEmptyStrings[-1].split("_")) return nonLayoutParameters
def _folderComparator(self, first, second): # parse parameters firstParameters = self.parseParameters(first) secondParameters = self.parseParameters(second) comparisonList = map(lambda x, y: x - y, firstParameters, secondParameters) result = reduce(lambda x,y: y if x == 0 else x, comparisonList) return int(np.sign(result))