From a945600015414372cd0bb4cf8730b9afe13ea872 Mon Sep 17 00:00:00 2001 From: Sourcery AI <> Date: Wed, 16 Feb 2022 10:11:55 +0000 Subject: [PATCH] 'Refactored by Sourcery' --- clcache/__main__.py | 3 +- clcache/caching.py | 127 +++++++++++++++++++------------------- clcache/storage.py | 11 ++-- tests/test_integration.py | 6 +- tests/test_performance.py | 7 ++- tests/test_unit.py | 6 +- 6 files changed, 77 insertions(+), 83 deletions(-) diff --git a/clcache/__main__.py b/clcache/__main__.py index dfc0e72..687d0cf 100644 --- a/clcache/__main__.py +++ b/clcache/__main__.py @@ -70,8 +70,7 @@ def __call__(self, parser, namespace, values, optional_string=None): class RemainderSetAction(argparse.Action): def __call__(self, parser, namespace, values, optional_string=None): - nonCommand = getattr(namespace, "non_command", None) - if nonCommand: + if nonCommand := getattr(namespace, "non_command", None): values.insert(0, nonCommand) setattr(namespace, self.dest, values) diff --git a/clcache/caching.py b/clcache/caching.py index 9c01dc9..8750945 100644 --- a/clcache/caching.py +++ b/clcache/caching.py @@ -115,14 +115,13 @@ def childDirectories(path, absolute=True): def normalizeBaseDir(baseDir): - if baseDir: - baseDir = os.path.normcase(baseDir) - if baseDir.endswith(os.path.sep): - baseDir = baseDir[0:-1] - return baseDir - else: + if not baseDir: # Converts empty string to None return None + baseDir = os.path.normcase(baseDir) + if baseDir.endswith(os.path.sep): + baseDir = baseDir[:-1] + return baseDir def getCachedCompilerConsoleOutput(path): @@ -184,7 +183,7 @@ def __init__(self, manifestSectionDir): self.lock = CacheLock.forPath(self.manifestSectionDir) def manifestPath(self, manifestHash): - return os.path.join(self.manifestSectionDir, manifestHash + ".json") + return os.path.join(self.manifestSectionDir, f'{manifestHash}.json') def manifestFiles(self): return filesBeneath(self.manifestSectionDir) @@ -298,10 +297,14 @@ def getManifestHash(compilerBinary, commandLine, sourceFile): for k in sorted(arguments.keys()): if k in argumentsWithPaths: commandLine.extend( - ["/" + k + collapseBasedirInCmdPath(arg) for arg in arguments[k]] + [ + f'/{k}{collapseBasedirInCmdPath(arg)}' + for arg in arguments[k] + ] ) + else: - commandLine.extend(["/" + k + arg for arg in arguments[k]]) + commandLine.extend([f'/{k}{arg}' for arg in arguments[k]]) commandLine.extend(collapseBasedirInCmdPath(arg) for arg in inputFiles) @@ -332,7 +335,7 @@ class CacheLock: WAIT_TIMEOUT_CODE = 0x00000102 def __init__(self, mutexName, timeoutMs): - self._mutexName = "Local\\" + mutexName + self._mutexName = f'Local\\{mutexName}' self._mutex = None self._timeoutMs = timeoutMs @@ -408,7 +411,7 @@ def hasEntry(self, key): def setEntry(self, key, artifacts): cacheEntryDir = self.cacheEntryDir(key) # Write new files to a temporary directory - tempEntryDir = cacheEntryDir + ".new" + tempEntryDir = f'{cacheEntryDir}.new' # Remove any possible left-over in tempEntryDir from previous executions rmtree(tempEntryDir, ignore_errors=True) ensureDirectoryExists(tempEntryDir) @@ -984,27 +987,26 @@ def getCompilerHash(compilerBinary): def getFileHashes(filePaths): - if "CLCACHE_SERVER" in os.environ: - pipeName = r"\\.\pipe\clcache_srv" - while True: - try: - with open(pipeName, "w+b") as f: - f.write("\n".join(filePaths).encode("utf-8")) - f.write(b"\x00") - response = f.read() - if response.startswith(b"!"): - raise pickle.loads(response[1:-1]) - return response[:-1].decode("utf-8").splitlines() - except OSError as e: - if ( - e.errno == errno.EINVAL - and windll.kernel32.GetLastError() == ERROR_PIPE_BUSY - ): - windll.kernel32.WaitNamedPipeW(pipeName, NMPWAIT_WAIT_FOREVER) - else: - raise - else: + if "CLCACHE_SERVER" not in os.environ: return [getFileHash(filePath) for filePath in filePaths] + pipeName = r"\\.\pipe\clcache_srv" + while True: + try: + with open(pipeName, "w+b") as f: + f.write("\n".join(filePaths).encode("utf-8")) + f.write(b"\x00") + response = f.read() + if response.startswith(b"!"): + raise pickle.loads(response[1:-1]) + return response[:-1].decode("utf-8").splitlines() + except OSError as e: + if ( + e.errno == errno.EINVAL + and windll.kernel32.GetLastError() == ERROR_PIPE_BUSY + ): + windll.kernel32.WaitNamedPipeW(pipeName, NMPWAIT_WAIT_FOREVER) + else: + raise def getFileHash(filePath, additionalData=None): @@ -1027,29 +1029,28 @@ def getStringHash(dataString): def expandBasedirPlaceholder(path): baseDir = normalizeBaseDir(os.environ.get("CLCACHE_BASEDIR")) - if path.startswith(BASEDIR_REPLACEMENT): - if not baseDir: - print( - "No CLCACHE_BASEDIR set, but found relative path " + path, - file=sys.stderr, - ) - sys.exit(1) - return path.replace(BASEDIR_REPLACEMENT, baseDir, 1) - else: + if not path.startswith(BASEDIR_REPLACEMENT): return path + if not baseDir: + print( + f'No CLCACHE_BASEDIR set, but found relative path {path}', + file=sys.stderr, + ) + + sys.exit(1) + return path.replace(BASEDIR_REPLACEMENT, baseDir, 1) def collapseBasedirToPlaceholder(path): baseDir = normalizeBaseDir(os.environ.get("CLCACHE_BASEDIR")) if baseDir is None: return path + assert path == os.path.normcase(path) + assert baseDir == os.path.normcase(baseDir) + if path.startswith(baseDir): + return path.replace(baseDir, BASEDIR_REPLACEMENT, 1) else: - assert path == os.path.normcase(path) - assert baseDir == os.path.normcase(baseDir) - if path.startswith(baseDir): - return path.replace(baseDir, BASEDIR_REPLACEMENT, 1) - else: - return path + return path def ensureDirectoryExists(path): @@ -1078,7 +1079,7 @@ def copyOrLink(srcFilePath, dstFilePath, writeCache=False): # If hardlinking fails for some reason (or it's not enabled), just # fall back to moving bytes around. Always to a temporary path first to # lower the chances of corrupting it. - tempDst = dstFilePath + ".tmp" + tempDst = f'{dstFilePath}.tmp' if "CLCACHE_COMPRESS" in os.environ: if "CLCACHE_COMPRESSLEVEL" in os.environ: @@ -1170,10 +1171,9 @@ def _parseBackslash(self): self._pos += 1 numBackslashes += 1 - followedByDoubleQuote = ( + if followedByDoubleQuote := ( self._pos < len(self._content) and self._content[self._pos] == '"' - ) - if followedByDoubleQuote: + ): self._token += "\\" * (numBackslashes // 2) if numBackslashes % 2 == 0: self._pos -= 1 @@ -1248,7 +1248,7 @@ def __len__(self): return len(self.name) def __str__(self): - return "/" + self.name + return f'/{self.name}' def __eq__(self, other): return type(self) == type(other) and self.name == other.name @@ -1328,11 +1328,14 @@ class CommandLineAnalyzer: @staticmethod def _getParameterizedArgumentType(cmdLineArgument): - # Sort by length to handle prefixes - for arg in CommandLineAnalyzer.argumentsWithParameterSorted: - if cmdLineArgument.startswith(arg.name, 1): - return arg - return None + return next( + ( + arg + for arg in CommandLineAnalyzer.argumentsWithParameterSorted + if cmdLineArgument.startswith(arg.name, 1) + ), + None, + ) @staticmethod def parseArgumentsAndInputFiles(cmdline): @@ -1435,10 +1438,11 @@ def analyze(cmdline: List[str]) -> Tuple[List[Tuple[str, str]], List[str]]: if objectFiles is None: # Generate from .c/.cpp filenames objectFiles = [ - os.path.join(prefix, basenameWithoutExtension(f)) + ".obj" + f'{os.path.join(prefix, basenameWithoutExtension(f))}.obj' for f, _ in inputFiles ] + printTraceStatement("Compiler source files: {}".format(inputFiles)) printTraceStatement("Compiler object file: {}".format(objectFiles)) return inputFiles, objectFiles @@ -1662,11 +1666,10 @@ def createManifestEntry(manifestHash, includePaths): def run(cache, compiler, compiler_args, env): printTraceStatement("Found real compiler binary at '{0!s}'".format(compiler)) - if "CLCACHE_DISABLE" in os.environ: - exit_code, out, err = invokeRealCompiler(compiler, compiler_args, env) - return exit_code, [out], [err] - else: + if "CLCACHE_DISABLE" not in os.environ: return processCompileRequest(cache, compiler, compiler_args, env) + exit_code, out, err = invokeRealCompiler(compiler, compiler_args, env) + return exit_code, [out], [err] cache = None @@ -1762,7 +1765,7 @@ def processCompileRequest(cache, compiler, args, env): def filterSourceFiles( cmdLine: List[str], sourceFiles: List[Tuple[str, str]] ) -> Iterator[str]: - setOfSources = set(sourceFile for sourceFile, _ in sourceFiles) + setOfSources = {sourceFile for sourceFile, _ in sourceFiles} skippedArgs = ("/Tc", "/Tp", "-Tp", "-Tc") return ( arg diff --git a/clcache/storage.py b/clcache/storage.py index 2943326..ac789e8 100644 --- a/clcache/storage.py +++ b/clcache/storage.py @@ -63,7 +63,7 @@ def connect(self, server): deserializer=python_memcache_deserializer, timeout=5, connect_timeout=5, - key_prefix=(getStringHash(self.fileStrategy.dir) + "_").encode("UTF-8"), + key_prefix=f'{getStringHash(self.fileStrategy.dir)}_'.encode("UTF-8"), ) # XX key_prefix ties fileStrategy cache to memcache entry # because tests currently the integration tests use this to start with clean cache @@ -213,8 +213,7 @@ def getEntry(self, key): if self.localCache.hasEntry(key): printTraceStatement("Getting object {} from local cache".format(key)) return self.localCache.getEntry(key) - remote = self.remoteCache.getEntry(key) - if remote: + if remote := self.remoteCache.getEntry(key): printTraceStatement("Getting object {} from remote cache".format(key)) return remote return None @@ -229,14 +228,12 @@ def setManifest(self, manifestHash, manifest): self.remoteCache.setManifest(manifestHash, manifest) def getManifest(self, manifestHash): - local = self.localCache.getManifest(manifestHash) - if local: + if local := self.localCache.getManifest(manifestHash): printTraceStatement( "{} local manifest hit for {}".format(self, manifestHash) ) return local - remote = self.remoteCache.getManifest(manifestHash) - if remote: + if remote := self.remoteCache.getManifest(manifestHash): with self.localCache.manifestLockFor(manifestHash): self.localCache.setManifest(manifestHash, remote) printTraceStatement( diff --git a/tests/test_integration.py b/tests/test_integration.py index 80f7c7d..1958648 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -156,10 +156,8 @@ def testRecompileObjectSetOtherDir(self): def testPipedOutput(self): def debugLinebreaks(text): - out = [] lines = text.splitlines(True) - for line in lines: - out.append(line.replace("\r", "").replace("\n", "")) + out = [line.replace("\r", "").replace("\n", "") for line in lines] return "\n".join(out) commands = [ @@ -845,7 +843,7 @@ def testOutput(self): clcache.Cache(tempDir) customEnv = self._createEnv(tempDir) cmd = CLCACHE_CMD + ["/nologo", "/EHsc", "/c"] - mpFlag = "/MP" + str(len(sources)) + mpFlag = f'/MP{len(sources)}' out = subprocess.check_output(cmd + [mpFlag] + sources, env=customEnv).decode("ascii") # print the output so that it shows up in py.test print(out) diff --git a/tests/test_performance.py b/tests/test_performance.py index 2446e52..afbcf19 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -45,9 +45,10 @@ def setUpClass(cls): os.path.join(ASSETS_DIR, 'concurrency', 'file{:02d}.cpp'.format(i+1)) ) - cls.sources = [] - for i in range(1, TestConcurrency.NUM_SOURCE_FILES+1): - cls.sources.append(os.path.join(ASSETS_DIR, 'concurrency', 'file{:02d}.cpp'.format(i))) + cls.sources = [ + os.path.join(ASSETS_DIR, 'concurrency', 'file{:02d}.cpp'.format(i)) + for i in range(1, TestConcurrency.NUM_SOURCE_FILES + 1) + ] def testConcurrentHitsScaling(self): with tempfile.TemporaryDirectory() as tempDir: diff --git a/tests/test_unit.py b/tests/test_unit.py index 23d9f2d..b42296e 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -521,10 +521,6 @@ def _testSourceFilesOk(self, cmdLine): except AnalysisError as err: if isinstance(err, NoSourceFileError): self.fail("analyze() unexpectedly raised an NoSourceFileError") - else: - # We just want to know if we got a proper source file. - # Other AnalysisErrors are ignored. - pass def _testFailure(self, cmdLine, expectedExceptionClass): self.assertRaises(expectedExceptionClass, lambda: CommandLineAnalyzer.analyze(cmdLine)) @@ -1133,7 +1129,7 @@ def assertEntrySizeIsCorrect(self, expectedSize): srcFilePath = os.path.join(self.testDir, "src") dstFilePath = os.path.join(self.testDir, "dst") with open(srcFilePath, "wb") as f: - for i in range(0, 999): + for i in range(999): f.write(b"%d" % i) copyOrLink(srcFilePath, dstFilePath, True) size = os.path.getsize(dstFilePath)