diff --git a/.coveragerc b/.coveragerc index d504abb..4d62152 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,7 @@ omit = */python?.?/* */site-packages/nose/* */opentuner/opentuner/* + */test/* + */ctree/tools/* + */ctree/visual/* + */ctree/metrics/* diff --git a/.gitignore b/.gitignore index 0eea557..53d2149 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ # C extensions *.so +*.o # Packages *.egg @@ -37,6 +38,7 @@ nosetests.xml .mr.developer.cfg .project .pydevproject +.idea # vim temp files .*.swp @@ -52,7 +54,17 @@ htmlcov # virtualenv subdirs venv-* +.venv # opentuner stuff opentuner.db opentuner.log + +# rope library +.ropeproject + +# compiled files +compiled/* + +*.sublime-project +*.sublime-workspace diff --git a/.travis.cfg b/.travis.cfg new file mode 100644 index 0000000..1fb0832 --- /dev/null +++ b/.travis.cfg @@ -0,0 +1,28 @@ +[jit] +COMPILE_PATH = ./compiled +CACHE = False + +[c] +CC = gcc +CFLAGS = -fPIC -O2 -std=c99 +LDFLAGS = + +[omp] +CC = gcc +CFLAGS = -fPIC -std=c99 -O2 -I/opt/intel/composerxe/include -fopenmp +LDFLAGS = + +[opencl] +CC = gcc +CFLAGS = -fPIC -std=c99 -O2 +# For Linux +LDFLAGS = -L/home/travis/AMDAPPSDK/lib/x86_64 -lOpenCL + +[log] +# maximum number of lines to show when programs are printed to the log +max_lines_per_source = 10 +pygments_style = vim + +[opentuner] +args = --quiet --no-dups +timeout = 3 diff --git a/.travis.yml b/.travis.yml index 92259b0..d5a50f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,90 +1,54 @@ +cache: + - apt + - pip language: python - python: - - "2.7" - - "3.2" - - "3.3" - + - '2.7' + - '3.4' env: global: - # encrypted OAuth token so Travis can commit docs back to Github - - secure: "QBB2KT4EFUdSkH9fjo5J/02zuZlD0FbVsKwYJgW6c4INp1UY/nx0nrsgjPSZQMD2HBztdfEZInugoVrOJwXBzWJ5Ioc19T9oYhnzaIF6oJRo1mTYDhragvdwiLfb0AyylGq7bgP4lgoMBtE1Oxauf0rKVEYiVeEhvHJup/di6A4=" - matrix: - - LLVM_VERSION=3.3 - + - secure: QBB2KT4EFUdSkH9fjo5J/02zuZlD0FbVsKwYJgW6c4INp1UY/nx0nrsgjPSZQMD2HBztdfEZInugoVrOJwXBzWJ5Ioc19T9oYhnzaIF6oJRo1mTYDhragvdwiLfb0AyylGq7bgP4lgoMBtE1Oxauf0rKVEYiVeEhvHJup/di6A4= before_install: - - sudo apt-get update -qq - - sudo apt-get install -qq llvm-$LLVM_VERSION - - + - if [ $TRAVIS_OS_NAME = "linux" ]; then + bash .travis/amd_sdk.sh; + tar -xjf AMD-SDK.tar.bz2; + AMDAPPSDK=${HOME}/AMDAPPSDK; + export OPENCL_VENDOR_PATH=${AMDAPPSDK}/etc/OpenCL/vendors; + mkdir -p ${OPENCL_VENDOR_PATH}; + sh AMD-APP-SDK*.sh --tar -xf -C ${AMDAPPSDK}; + echo libamdocl64.so > ${OPENCL_VENDOR_PATH}/amdocl64.icd; + export PYCL_OPENCL=${AMDAPPSDK}/lib/x86_64/libOpenCL.so; + export LD_LIBRARY_PATH=${AMDAPPSDK}/lib/x86_64:${LD_LIBRARY_PATH}; + chmod +x ${AMDAPPSDK}/bin/x86_64/clinfo; + ${AMDAPPSDK}/bin/x86_64/clinfo; + find ${AMDAPPSDK} -name libOpenCL*; + fi; + - sudo apt-get install -qq opencl-headers install: - # make 'x.y' version string - export PYTHON_VERSION=$(python -c 'import sys; print(sys.version_info[0:2])') - - # coverage and doc generator - - pip install numpy Sphinx coveralls coverage nose pygments + - pip install -r requirements.txt + - if [ $PY_MAJOR_MINOR \< "3.0" ]; then pip install opentuner; fi + - pip install coverage nose pycl + - cp .travis.cfg ctree/defaults.cfg - nosetests --version - coverage --version - - # install llvmpy - - git clone git://github.com/llvmpy/llvmpy.git ${HOME}/llvmpy - - cd ${HOME}/llvmpy - - LLVM_CONFIG_PATH=/usr/bin/llvm-config-$LLVM_VERSION python setup.py install - - # install opentuner - - git clone https://github.com/mbdriscoll/opentuner.git ${HOME}/opentuner - - cd ${HOME}/opentuner - - sudo apt-get install `cat debian-packages-deps | tr '\n' ' '` - - "if [[ \"x$PYTHON_VERSION\" -eq \"x(2.7)\" ]]; then pip install -r python-packages; fi" - - export PYTHONPATH=`pwd`:$PYTHONPATH - - # install ctree via setup.py - - cd ${TRAVIS_BUILD_DIR} - python setup.py install - + - env script: - - # run test suite from home directory to verify installation - - cd ${HOME} - - nosetests --where=${TRAVIS_BUILD_DIR}/test - - # run test suite again from build dir to get coverage info - cd ${TRAVIS_BUILD_DIR} - - nosetests --verbose --with-coverage --cover-package=ctree --cover-min-percentage=90 --cover-erase - - + - nosetests --verbose --with-coverage --cover-package=ctree --cover-min-percentage=87 + --cover-erase after_success: - - # return early if not building ucb-sejits/ctree - - "if [[ \"x${TRAVIS_REPO_SLUG}\" != 'xucb-sejits/ctree' ]]; then echo 'skipping coveralls/sphinx for non ucb-sejits/ctree builds.'; exit 0; fi" - - # only build docs using Python 2.7 - - "if [[ \"x$PYTHON_VERSION\" != \"x(2, 7)\" ]]; then echo 'Not Python 2.7; skipping doc build.'; exit 0; fi" - - # publish coverage report - - coveralls - - # build documentation - - make -C doc html - - # merge documentation - - git clone "https://github.com/ucb-sejits/ctree-docs.git" ${HOME}/ctree-docs - - cd ${HOME}/ctree-docs - - git fetch origin gh-pages - - git checkout gh-pages - - rsync -a ${TRAVIS_BUILD_DIR}/doc/_build/html/ ./ - - git add . - - git status - - # commit documentation - - git config --global user.name 'Ctree Doc Bot' - - git config --global user.email 'mbdriscoll+ctreeoauth@gmail.com' - - git commit -m "Updating documentation from Travis Build ${TRAVIS_BUILD_ID}." - - # set up oauth for push - - git config credential.helper "store --file=.git/credentials" - - echo "https://${GH_TOKEN}:x-oauth-basic@github.com" > .git/credentials - - # commit new docs to ctree-docs - - git push origin gh-pages + - curl -X POST http://readthedocs.org/build/ctree +notifications: + slack: ucb-sejits:cPZxBunxagWZ763mcsIXOV0W +deploy: + provider: pypi + user: leonardt + password: + secure: SMiyQflUvfG0M8bR07Sri8VXnPSFKprNxA3RF7sljk99Aj9BuuuBRLkcOhkYtIRYfgHUSEnFeYYe+rb8y6BV/LnulCQiw9bCIqmPY9IYGy63DNjUGxh65MyO9HDjwz4hi+4endwZTXaUL3X4de9Xk3NnDhHISiLd7WymR9YQ7eE= + on: + tags: true + all_branches: true + repo: ucb-sejits/ctree diff --git a/.travis/amd_sdk.sh b/.travis/amd_sdk.sh new file mode 100644 index 0000000..f5fea98 --- /dev/null +++ b/.travis/amd_sdk.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Original script from https://github.com/gregvw/amd_sdk/ + +# Location from which get nonce and file name from +URL="http://developer.amd.com/tools-and-sdks/opencl-zone/opencl-tools-sdks/amd-accelerated-parallel-processing-app-sdk/" +URLDOWN="http://developer.amd.com/amd-license-agreement-appsdk/" + +NONCE1_STRING='name="amd_developer_central_downloads_page_nonce"' +FILE_STRING='name="f"' +POSTID_STRING='name="post_id"' +NONCE2_STRING='name="amd_developer_central_nonce"' + +#For newest FORM=`wget -qO - $URL | sed -n '/download-2/,/64-bit/p'` +FORM=`wget -qO - $URL | sed -n '/download-5/,/64-bit/p'` + +# Get nonce from form +NONCE1=`echo $FORM | awk -F ${NONCE1_STRING} '{print $2}'` +NONCE1=`echo $NONCE1 | awk -F'"' '{print $2}'` +echo $NONCE1 + +# get the postid +POSTID=`echo $FORM | awk -F ${POSTID_STRING} '{print $2}'` +POSTID=`echo $POSTID | awk -F'"' '{print $2}'` +echo $POSTID + +# get file name +FILE=`echo $FORM | awk -F ${FILE_STRING} '{print $2}'` +FILE=`echo $FILE | awk -F'"' '{print $2}'` +echo $FILE + +FORM=`wget -qO - $URLDOWN --post-data "amd_developer_central_downloads_page_nonce=${NONCE1}&f=${FILE}&post_id=${POSTID}"` + +NONCE2=`echo $FORM | awk -F ${NONCE2_STRING} '{print $2}'` +NONCE2=`echo $NONCE2 | awk -F'"' '{print $2}'` +echo $NONCE2 + +wget --content-disposition --trust-server-names $URLDOWN --post-data "amd_developer_central_nonce=${NONCE2}&f=${FILE}" -O AMD-SDK.tar.bz2; diff --git a/README.md b/README.md index db0f5fb..49d37ad 100644 --- a/README.md +++ b/README.md @@ -7,3 +7,18 @@ See the [website](http://ucb-sejits.github.io/ctree/) or [documentation](https:/ [![Build Status](https://travis-ci.org/ucb-sejits/ctree.png?branch=master)](https://travis-ci.org/ucb-sejits/ctree) [![Coverage Status](https://coveralls.io/repos/ucb-sejits/ctree/badge.png)](https://coveralls.io/r/ucb-sejits/ctree) +[![Documentation Status](https://readthedocs.org/projects/ctree/badge/?version=latest)](https://readthedocs.org/projects/ctree/?badge=latest) + +Quick Install +------------- +```shell +pip install ctree +``` +For OpenCL support, install the pycl package. +```shell +pip install pycl +``` + +Development +----------- +[See the wiki](https://github.com/ucb-sejits/ctree/wiki) diff --git a/ctree/__init__.py b/ctree/__init__.py index 470fd92..bbd121a 100644 --- a/ctree/__init__.py +++ b/ctree/__init__.py @@ -5,6 +5,8 @@ """ from __future__ import print_function + + # --------------------------------------------------------------------------- # explicit version check @@ -54,11 +56,15 @@ else: from io import StringIO as Memfile +from ctree.util import highlight + CONFIGFILE = Memfile() CONFIG.write(CONFIGFILE) CONFIG_TXT = CONFIGFILE.getvalue() -LOG.info("using configuration:\n%s", CONFIG_TXT) +LOG.info("using configuration:\n%s", highlight(CONFIG_TXT, language='ini')) CONFIGFILE.close() +if CONFIG.has_option('log','level'): + logging.basicConfig(level=getattr(logging,CONFIG.get('log','level'))) # --------------------------------------------------------------------------- @@ -68,7 +74,7 @@ import collections -class Counter(object): +class LogInfo(object): """Tracks events, reports counts upon garbage collections.""" def __init__(self): @@ -86,9 +92,44 @@ def report(self): LOG.info("execution statistics: (((\n%s)))", key_values_string) -STATS = Counter() +STATS = LogInfo() atexit.register(STATS.report) +#---------------------------------------------------------------------------- +#Temporary directory stuff +import tempfile +import shutil +import os.path + +if CONFIG.getboolean('jit', 'CACHE'): + STATS.log("recognized that caching is enabled") +else: + STATS.log("recognized that caching is disabled") + +if not CONFIG.getboolean('jit', 'CACHE'): + compile_path_old = CONFIG.get('jit', 'COMPILE_PATH') + temporary_path = tempfile.mkdtemp() + CONFIG.set('jit', 'COMPILE_PATH', temporary_path) + + def reset(): + CONFIG.set('jit', 'COMPILE_PATH', compile_path_old) + if(os.path.isdir(temporary_path)): + shutil.rmtree(temporary_path) + + atexit.register(reset) + +# Registries for type-based logic in extension packages. +_TYPE_CODEGENERATORS = {} +_TYPE_RECOGNIZERS = {} + +OCL_ENABLED = True +try: + import pycl +except ImportError: + OCL_ENABLED = False + +import ctree.np + import ast import inspect import ctree.frontend @@ -106,13 +147,15 @@ def ipython_show_ast(tree): converts tree in place to a dot format then renders that into a png file """ + import ctree.dotgen return DotManager.dot_ast_to_image(tree) -def browser_show_ast(tree, file_name): +def browser_show_ast(tree, file_name=None): """ convenience method to display an AST in ipython converts tree in place to a dot format then renders that into a png file """ - return DotManager.dot_ast_to_browser(tree, file_name) \ No newline at end of file + import ctree.dotgen + return DotManager.dot_ast_to_browser(tree, file_name) diff --git a/ctree/analyses.py b/ctree/analyses.py index 841b356..ee3aade 100644 --- a/ctree/analyses.py +++ b/ctree/analyses.py @@ -45,23 +45,3 @@ def visit(self, node): if not isinstance(node, CtreeNode): raise AstValidationError("Expected a pure C ast, but found a non-CtreeNode: %s." % node) self.generic_visit(node) - - -class VerifyParentPointers(NodeVisitor): - """ - Checks that parent pointers are set correctly, and throws - an AstValidationError if they're not. - """ - - def _check(self, child, parent): - """throw if child.parent is not the actual parent""" - if child.parent != parent: - raise AstValidationError("Expect parent of %s to be %s, but got %s instead." % - (type(child), type(parent), type(child.parent))) - - def generic_visit(self, node): - for _, value in ast.iter_fields(node): - for child in flatten(value): - if isinstance(child, ast.AST): - self._check(child, node) - self.visit(child) diff --git a/ctree/c/__init__.py b/ctree/c/__init__.py index e69de29..04500aa 100644 --- a/ctree/c/__init__.py +++ b/ctree/c/__init__.py @@ -0,0 +1,60 @@ +import types +import ctypes +import _ctypes +import sys + +from ctree.types import ( + codegen_type, + register_type_recognizers, + register_type_codegenerators, +) + +#Py2 and Py3 common types + +register_type_recognizers( + { + int: ctypes.c_long, + bool: ctypes.c_bool, + float: ctypes.c_double, + str: lambda t: ctypes.c_char(t.encode()) if len(t) == 1 else ctypes.c_char_p(t.encode()), + type(None): lambda t: None + } +) + +import sys + +if sys.maxsize > 2 ** 32: + X64_BIT = True +else: + # Python alias c_int to c_long on 32 bit platforms + X64_BIT = False + +register_type_codegenerators({ + ctypes.c_short: lambda t: "short", + ctypes.c_int: lambda t: "int", + ctypes.c_long: lambda t: "long" if X64_BIT else "int", + ctypes.c_float: lambda t: "float", + ctypes.c_double: lambda t: "double", + ctypes.c_char: lambda t: "char", + ctypes.c_char_p: lambda t: "char*", + ctypes.c_void_p: lambda t: "void*", + ctypes.c_bool: lambda t: "bool", + ctypes.c_ulong: lambda t: "size_t", + ctypes.c_uint32: lambda t: "uint32_t", + type(None): lambda n: "void", + + _ctypes.Array: lambda ct: "%s*" % codegen_type(ct._type_()), + _ctypes._Pointer: lambda ct: "%s*" % codegen_type(ct._type_()), + +}) + +#register version specific nodes + +if sys.version_info >= (3, 0): + pass + +else: + + register_type_recognizers({ + long: ctypes.c_long + }) diff --git a/ctree/c/codegen.py b/ctree/c/codegen.py index 5b03b3b..4fb8a4d 100644 --- a/ctree/c/codegen.py +++ b/ctree/c/codegen.py @@ -4,24 +4,24 @@ from ctree.codegen import CodeGenVisitor from ctree.c.nodes import Op -from ctree.c.types import Ptr, get_ctree_type +from ctree.types import codegen_type, get_suffix from ctree.precedence import UnaryOp, BinaryOp, TernaryOp, Cast from ctree.precedence import get_precedence, is_left_associative +from numbers import Number -class CCodeGen(CodeGenVisitor): +from ctree.nodes import CommonCodeGen + +class CCodeGen(CommonCodeGen): """ Manages generation of C code. """ - def _requires_parentheses(self, node): + def _requires_parentheses(self, parent, node): """ - Return True if the current precedence is less than the - parent precedence. If the precedences are equal, check whether the - node's orientation to the parent matches associativity. If it doesn't, - enclose with parentheses. + Returns node as a string, optionally with parentheses around it if + needed to enforce precendence rules. """ - parent = getattr(node, 'parent', None) if isinstance(node, (UnaryOp, BinaryOp, TernaryOp)) and\ isinstance(parent, (UnaryOp, BinaryOp, TernaryOp, Cast)): prec = get_precedence(node) @@ -39,44 +39,53 @@ def _requires_parentheses(self, node): # ------------------------------------------------------------------------- # visitor methods + def visit_MultiNode(self, node): + return self._genblock(node.body, insert_curly_brackets=False, increase_indent=False) + def visit_FunctionDecl(self, node): params = ", ".join(map(str, node.params)) - s = "" + s = [] + for attrib in node.attributes: + s.append("__attribute__ (({}))".format(attrib)) if node.kernel: - s += "__kernel " + s.append("__kernel") if node.static: - s += "static " + s.append("static") if node.inline: - s += "inline " - s += "%s %s(%s)" % (node.return_type, node.name, params) + s.append("inline") + s.append("%s %s(%s)" % (codegen_type(node.return_type), node.name, params)) if node.defn: - s += " %s" % self._genblock(node.defn) - return s + s.append("%s" % self._genblock(node.defn)) + return " ".join(s) def visit_UnaryOp(self, node): + op = self._parenthesize(node, node.op) + arg = self._parenthesize(node, node.arg) if isinstance(node.op, (Op.PostInc, Op.PostDec)): - s = "%s %s" % (node.arg, node.op) + return "%s %s" % (arg, op) else: - s = "%s %s" % (node.op, node.arg) - return self._parentheses(node) % s + return "%s %s" % (op, arg) def visit_BinaryOp(self, node): + left = self._parenthesize(node, node.left) + right = self._parenthesize(node, node.right) if isinstance(node.op, Op.ArrayRef): - s = "%s[%s]" % (node.left, node.right) + return "%s[%s]" % (left, right) else: - s = "%s %s %s" % (node.left, node.op, node.right) - return self._parentheses(node) % s + return "%s %s %s" % (left, node.op, right) def visit_AugAssign(self, node): return "%s %s= %s" % (node.target, node.op, node.value) def visit_TernaryOp(self, node): - s = "%s ? %s : %s" % (node.cond, node.then, node.elze) - return self._parentheses(node) % s + cond = self._parenthesize(node, node.cond) + then = self._parenthesize(node, node.then) + elze = self._parenthesize(node, node.elze) + return "%s ? %s : %s" % (cond, then, elze) def visit_Cast(self, node): - s = "(%s) %s" % (node.type, node.value) - return self._parentheses(node) % s + value = self._parenthesize(node, node.value) + return "(%s) %s" % (codegen_type(node.type), value) def visit_Constant(self, node): if isinstance(node.value, str): @@ -90,10 +99,14 @@ def visit_SymbolRef(self, node): s += "__global " if node._local: s += "__local " + if node._static: + s += "static " if node._const: s += "const " - if node.type: - s += "%s " % node.type + if node.type is not None: + s += "%s " % codegen_type(node.type) + if node._restrict: + s += "restrict " return "%s%s" % (s, node.name) def visit_Block(self, node): @@ -123,7 +136,10 @@ def visit_DoWhile(self, node): def visit_For(self, node): body = self._genblock(node.body) - return "for (%s; %s; %s) %s" % (node.init, node.test, node.incr, body) + s = "" + if node.pragma: + s += "#pragma %s\n" % node.pragma + self._tab() + return s + "for (%s; %s; %s) %s" % (node.init, node.test, node.incr, body) def visit_FunctionCall(self, node): args = ", ".join(map(str, node.args)) @@ -136,53 +152,30 @@ def visit_CFile(self, node): stmts = self._genblock(node.body, insert_curly_brackets=False, increase_indent=False) return '// %s' % (node.get_filename(), stmts) - def visit_Void(self, node): - return "void" - - def visit_Char(self, node): - return "char" - - def visit_UChar(self, node): - return "unsigned char" - - def visit_Short(self, node): - return "short" - - def visit_UShort(self, node): - return "unsigned short" - - def visit_Int(self, node): - return "int" - - def visit_UInt(self, node): - return "unsigned int" - - def visit_Long(self, node): - return "long" - - def visit_ULong(self, node): - return "unsigned long" + def visit_ArrayDef(self, node): + return "%s[%s] = " % (node.target, node.size) + self.visit(node.body) - def visit_Float(self, node): - return "float" + def visit_Break(self, node): + return 'break' - def visit_Double(self, node): - return "double" + def visit_Continue(self, node): + return 'continue' - def visit_LongDouble(self, node): - return "long double" + def visit_Array(self, node): + return "{%s}" % ', '.join([i.codegen() for i in node.body]) - def visit_Ptr(self, node): - base = node.base_type.codegen() - return "%s*" % base + def visit_Hex(self, node): + return hex(node.value) + get_suffix(node.ctype) - def visit_NdPointer(self, node): - inner_type = get_ctree_type(node.ptr._dtype_) - return "%s" % Ptr(inner_type).codegen() + def visit_Number(self, node): + return str(node.value) + get_suffix(node.ctype) - def visit_FILE(self, node): - return "FILE" + def visit_Attribute(self, node): + s = self.visit(node.target) + return "{target} __attribute__({items})".format(target=s, items=", ".join(node.attributes)) - def visit_ArrayDef(self, node): - body = ", ".join(map(str, node.body)) - return "{ %s }" % body + def visit_Pragma(self, node): + stuff = self._genblock(node.body, insert_curly_brackets=node.braces) + if node.braces: + stuff = '\n\t'.join(stuff.split("\n")) + return '#pragma ' + node.pragma + '\n' + stuff diff --git a/ctree/c/dotgen.py b/ctree/c/dotgen.py index 672c8cf..7fdecf8 100644 --- a/ctree/c/dotgen.py +++ b/ctree/c/dotgen.py @@ -2,21 +2,29 @@ DOT generator for C constructs. """ -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenLabeller +from ctree.types import codegen_type -class CDotGen(DotGenVisitor): +class CDotGenLabeller(DotGenLabeller): """ Manages generation of DOT. """ - def label_SymbolRef(self, node): - if node.type: - return r"%s %s" % (node.type, node.name) - else: - return r"%s" % (node.name) + def visit_SymbolRef(self, node): + s = r"" + if node._global: + s += r"__global " + if node._local: + s += r"__local " + if node._const: + s += r"__const " + if node.type is not None: + s += r"%s " % codegen_type(node.type) + s += r"%s" % node.name + return s - def label_FunctionDecl(self, node): + def visit_FunctionDecl(self, node): s = r"" if node.static: s += r"static " @@ -24,23 +32,26 @@ def label_FunctionDecl(self, node): s += r"inline " if node.kernel: s += r"__kernel " - s += r"%s %s(...)" % (node.return_type, node.name) + s += r"%s %s(...)" % (codegen_type(node.return_type), node.name) return s - def label_Constant(self, node): + def visit_Constant(self, node): return str(node.value) - def label_String(self, node): + def visit_String(self, node): return r'\" \"'.join(node.values) - def label_CFile(self, node): + def visit_CFile(self, node): return node.get_filename() - def label_NdPointer(self, node): + def visit_NdPointer(self, node): s = "dtype: %s\n" % node.ptr.dtype s += "ndim, shape: %s, %s\n" % (node.ptr.ndim, node.ptr.shape) s += "flags: %s" % node.ptr.flags return s - def label_BinaryOp(self, node): + def visit_BinaryOp(self, node): + return type(node.op).__name__ + + def visit_UnaryOp(self, node): return type(node.op).__name__ diff --git a/ctree/c/nodes.py b/ctree/c/nodes.py index 444e88e..0441bf6 100644 --- a/ctree/c/nodes.py +++ b/ctree/c/nodes.py @@ -3,14 +3,20 @@ """ import os +import types import subprocess import logging log = logging.getLogger(__name__) +from ctypes import CFUNCTYPE from ctree.nodes import CtreeNode, File -from ctree.util import singleton, highlight +import ctree +from ctree.util import singleton, highlight, truncate +from ctree.types import get_ctype, get_common_ctype, get_c_type_from_numpy_dtype +import hashlib +import ctypes class CNode(CtreeNode): @@ -18,62 +24,133 @@ class CNode(CtreeNode): def codegen(self, indent=0): from ctree.c.codegen import CCodeGen + from ctree.transforms import DeclarationFiller return CCodeGen(indent).visit(self) - def _to_dot(self): - from ctree.c.dotgen import CDotGen + def label(self): + from ctree.c.dotgen import CDotGenLabeller + + return CDotGenLabeller().visit(self) + + def __add__(self, other): + return Add(self, other) + + def __neg__(self): + return BitNot(self) + + def __sub__(self, other): + return Sub(self, other) + + def __or__(self, other): + return BitOr(self, other) + + def __and__(self, other): + return BitAnd(self, other) + + def __xor__(self, other): + return BitXor(self, other) + + def __lshift__(self, other): + if isinstance(other, int): + return BitShL(self, Constant(other)) + return BitShL(self, other) + + def __rshift__(self, other): + if isinstance(other, int): + return BitShR(self, Constant(other)) + return BitShR(self, other) + + def __mul__(self, other): + return Mul(self, other) + + def __div__(self, other): + return Div(self, other) + + __truediv__ = __div__ + + def __mod__(self, other): + return Mod(self, other) + - return CDotGen().visit(self) class CFile(CNode, File): """Represents a .c file.""" + _ext = "c" - def __init__(self, name="generated", body=None): - if not body: - body = [] - super(CFile, self).__init__(name, body) - self._ext = "c" + def __init__(self, name="generated", body=None, config_target='c', path=None): + CNode.__init__(self) + File.__init__(self, name, body, path) + self.config_target = config_target def get_bc_filename(self): return "%s.bc" % self.name - def _compile(self, program_text, compilation_dir): - import ctree - from ctree.util import truncate - - c_src_file = os.path.join(compilation_dir, self.get_filename()) - ll_bc_file = os.path.join(compilation_dir, self.get_bc_filename()) - log.info("file for generated C: %s", c_src_file) - log.info("file for generated LLVM: %s", ll_bc_file) - - # syntax-highlight and print C program - highlighted = highlight(program_text, 'c') - log.info("generated C program: (((\n%s\n)))", highlighted) + def get_so_filename(self): + return "{}.so".format(self.name) + + def _compile(self, program_text): + # print(repr(self.path), repr(self.get_filename())) + c_src_file = os.path.join(self.path, self.get_filename()) + so_file = os.path.join(self.path, self.get_so_filename()) + program_hash = hashlib.sha512(program_text.strip().encode()).hexdigest() + so_file_exists = os.path.exists(so_file) + old_hash = self.program_hash + hash_match = old_hash == program_hash + log.debug("Old hash: %s \n New hash: %s", old_hash, program_hash) + recreate_c_src = program_text and program_text != self.empty and not hash_match + recreate_so = recreate_c_src or not so_file_exists + + log.debug("RECREATE_C_SRC: %s \t RECREATE_so: %s \t HASH_MATCH: %s", + recreate_c_src, recreate_so, hash_match) + + if not program_text: + log.debug("Program not found. Attempting to use cached version") + + #create c_src + if recreate_c_src: + with open(c_src_file, 'w') as c_file: + c_file.write(program_text) + log.info("file for generated C: %s", c_src_file) + # syntax-highlight and print C program + highlighted = highlight(program_text, 'c') + log.info("generated C program: (((\n%s\n)))", highlighted) + self.program_hash = program_hash + + + #create ll_bc_file + if recreate_so: + # call clang to generate LLVM bitcode file + log.debug('Regenerating so.') + CC = ctree.CONFIG.get(self.config_target, 'CC') + CFLAGS = ctree.CONFIG.get(self.config_target, 'CFLAGS') + LDFLAGS = ctree.CONFIG.get(self.config_target, 'LDFLAGS') + compile_cmd = "%s -shared %s -o %s %s %s" % (CC, CFLAGS, so_file, + c_src_file, LDFLAGS) + log.info("compilation command: %s", compile_cmd) + subprocess.check_call(compile_cmd, shell=True) + # log.info("file for generated so: %s", so_file) + + #use cached version otherwise + if not (so_file_exists or recreate_so): + raise NotImplementedError('No Cached version found') - # write program text to C file - with open(c_src_file, 'w') as c_file: - c_file.write(program_text) - - # call clang to generate LLVM bitcode file - CC = ctree.CONFIG.get('jit', 'CC') - CFLAGS = ctree.CONFIG.get('jit', 'CFLAGS') - compile_cmd = "%s -emit-llvm %s -o %s -c %s" % (CC, CFLAGS, ll_bc_file, c_src_file) - log.info("compilation command: %s", compile_cmd) - subprocess.check_call(compile_cmd, shell=True) # load llvm bitcode - import llvm.core + # import llvm.core + # import llvmlite.binding as llvm - with open(ll_bc_file, 'rb') as bc: - ll_module = llvm.core.Module.from_bitcode(bc) + # with open(ll_bc_file, 'rb') as bc: + # ll_module = llvm.module.parse_bitcode(bc.read()) # syntax-highlight and print LLVM program - highlighted = highlight(str(ll_module), 'llvm') - log.debug("generated LLVM Program: (((\n%s\n)))", highlighted) + #preserve_src_drhighlighted = highlight(str(ll_module), 'llvm') + #log.debug("generated LLVM Program: (((\n%s\n)))", highlighted) + + return so_file + - return ll_module class Statement(CNode): @@ -84,11 +161,6 @@ class Statement(CNode): class Expression(CNode): """Cite me.""" - def get_type(self): - from ctree.c.types import CTypeFetcher - - return CTypeFetcher().visit(self) - class Return(Statement): """Section B.2.3 6.6.6 line 4.""" @@ -113,6 +185,7 @@ def __init__(self, cond=None, then=None, elze=None): class While(Statement): """Cite me.""" _fields = ['cond', 'body'] + _requires_semicolon = lambda self: False def __init__(self, cond=None, body=None): self.cond = cond @@ -131,12 +204,16 @@ def __init__(self, body=None, cond=None): class For(Statement): _fields = ['init', 'test', 'incr', 'body'] + _requires_semicolon = lambda self: False - def __init__(self, init=None, test=None, incr=None, body=None): + def __init__(self, init=None, test=None, incr=None, body=None, pragma=None): self.init = init self.test = test self.incr = incr + if body is None: + body = [] self.body = body + self.pragma = pragma super(For, self).__init__() @@ -161,11 +238,25 @@ class Literal(Expression): class Constant(Literal): """Section B.1.4 6.1.3.""" + _fields = ['value'] def __init__(self, value=None): self.value = value super(Constant, self).__init__() + def get_type(self, env=None): + return get_ctype(self.value) + +class Number(Constant): + def __init__(self, value, ctype=ctypes.c_uint32): + self.ctype = ctype + super(Number, self).__init__(value) + def get_type(self): + return self.ctype + +class Hex(Number): + pass + class Block(Statement): """Cite me.""" @@ -175,6 +266,16 @@ def __init__(self, body=None): self.body = body if body else [] super(Block, self).__init__() + def _requires_semicolon(self): + return False + + +class MultiNode(Block): + """ + Some Python nodes need to be translated to a block of nodes but Visitors can't do that. + """ + + class String(Literal): """Cite me.""" @@ -187,19 +288,25 @@ def __init__(self, *values): class SymbolRef(Literal): """Cite me.""" _next_id = 0 + _fields = ['name','type'] def __init__(self, name=None, sym_type=None, _global=False, - _local=False, _const=False): + _local=False, _const=False, _static=False, _restrict=False): """ Create a new symbol with the given name. If a declaration type is specified, the symbol is considered a declaration and unparsed with the type. """ self.name = name + + if sym_type is not None: + assert not isinstance(sym_type, type) self.type = sym_type self._global = _global self._local = _local self._const = _const + self._static = _static + self._restrict = _restrict super(SymbolRef, self).__init__() def set_global(self, value=True): @@ -214,6 +321,14 @@ def set_const(self, value=True): self._const = value return self + def set_static(self, value=True): + self._static = value + return self + + def set_restrict(self, value=True): + self._restrict = value + return self + @classmethod def unique(cls, name="name", sym_type=None): """ @@ -229,14 +344,11 @@ def copy(self, declare=False): else: return SymbolRef(self.name) - def get_ctype(self): - return self.type.as_ctype() - class FunctionDecl(Statement): """Cite me.""" - _fields = ['params', 'defn'] + _fields = ['params', 'defn', 'attributes'] - def __init__(self, return_type=None, name=None, params=None, defn=None): + def __init__(self, return_type=None, name=None, params=None, defn=None, attributes=()): self.return_type = return_type self.name = name self.params = params if params else [] @@ -244,12 +356,29 @@ def __init__(self, return_type=None, name=None, params=None, defn=None): self.inline = False self.static = False self.kernel = False + self.attributes = attributes super(FunctionDecl, self).__init__() - def get_type(self): - from ctree.c.types import FuncType - arg_types = [p.get_type() for p in self.params] - return FuncType(self.return_type, arg_types) + def get_type(self, env=None): + type_sig = [] + + # return type + if self.return_type is None: + type_sig.append(self.return_type) + else: + assert not isinstance(self.return_type, type), \ + "Expected a ctypes instance or None, got %s (%s)." % \ + (self.return_type, type(self.return_type)) + type_sig.append( type(self.return_type) ) + + # parameter types + for param in self.params: + assert not isinstance(param.type, type), \ + "Expected a ctypes instance or None, got %s (%s)." % \ + (param.type, type(param.type)) + type_sig.append( type(param.type) ) + + return CFUNCTYPE(*type_sig) def set_inline(self, value=True): self.inline = value @@ -263,14 +392,6 @@ def set_kernel(self, value=True): self.kernel = value return self - def set_typesig(self, func_type): - from ctree.c.types import FuncType - assert isinstance(func_type, FuncType) - self.return_type = func_type.return_type - for sym, ty in zip(self.params, func_type.arg_types): - sym.type = ty - return self - class UnaryOp(Expression): """Cite me.""" @@ -284,7 +405,7 @@ def __init__(self, op=None, arg=None): class BinaryOp(Expression): """Cite me.""" - _fields = ['left', 'right'] + _fields = ['left', 'op', 'right'] def __init__(self, left=None, op=None, right=None): self.left = left @@ -292,6 +413,38 @@ def __init__(self, left=None, op=None, right=None): self.right = right super(BinaryOp, self).__init__() + def get_type(self, env=None): + if isinstance(self.op, Op.ArrayRef): + if isinstance(self.left, SymbolRef) and env is not None \ + and env._has_key(self.left.name): + type = env._lookup(self.left.name)._dtype_ + return get_c_type_from_numpy_dtype(type)() + + # FIXME: integer promotions and stuff like that + if hasattr(self.left, 'get_type'): + left_type = self.left.get_type() + elif isinstance(self.left, SymbolRef) and env is not None \ + and env._has_key(self.left.name): + left_type = env._lookup(self.left.name) + elif hasattr(self.left, 'type'): + left_type = self.left.type + else: + left_type = None + if hasattr(self.right, 'get_type'): + right_type = self.right.get_type() + elif isinstance(self.right, SymbolRef) and env is not None \ + and env._has_key(self.right.name): + right_type = env._lookup(self.right.name) + elif hasattr(self.right, 'type'): + right_type = self.right.type + else: + right_type = None + if isinstance(self.op, Op.ArrayRef): + ptr_type = left_type._type_ + return ptr_type() if hasattr(ptr_type, '__call__') else left_type + return get_common_ctype(filter(lambda x: x is not None, [right_type, + left_type])) + class AugAssign(Expression): """Cite me.""" @@ -303,7 +456,6 @@ def __init__(self, target=None, op=None, value=None): self.value = value super(AugAssign, self).__init__() - class TernaryOp(Expression): """Cite me.""" _fields = ['cond', 'then', 'elze'] @@ -327,16 +479,42 @@ def __init__(self, sym_type=None, value=None): class ArrayDef(Expression): """doc""" - _fields = ['body'] + _fields = ['target', 'size', 'body'] - def __init__(self, body=None): + def __init__(self, target=None, size=None, body=None): + self.target = target + self.size = size self.body = body if body else [] super(ArrayDef, self).__init__() +class Array(Expression): + _fields = ['type', 'size', 'body'] + + def __init__(self, type=None, size=None, body=None): + self.body = body or [] + self.size = size or len(self.body) + self.type = type + super(Array, self).__init__() + + def get_type(self, env=None): + return self.type + +class Break(Statement): + _requires_semicolon = lambda self : True + +class Continue(Statement): + _requires_semicolon = lambda self : True + +class Pass(Statement): + _requires_semicolon = lambda self: False + @singleton class Op: class _Op(object): + def __init__(self): + self._force_parentheses = False + def __str__(self): return self._c_str @@ -440,7 +618,7 @@ class Assign(_Op): _c_str = "=" class ArrayRef(_Op): - _c_str = "??" + _c_str = "[]" # --------------------------------------------------------------------------- @@ -618,3 +796,19 @@ def BitShLAssign(a, b): def BitShRAssign(a, b): return AugAssign(a, Op.BitShR(), b) + +#--- NonStandard nodes + +class Attribute(CNode): + _fields = ['target'] + _force_parentheses = False + + def __init__(self, target, attributes=()): + self.target = target + self.attributes = attributes + +class Pragma(Block): + def __init__(self, pragma, body=(), braces=False): + self.body = body + self.pragma = pragma + self.braces = braces diff --git a/ctree/c/types.py b/ctree/c/types.py deleted file mode 100644 index 295f889..0000000 --- a/ctree/c/types.py +++ /dev/null @@ -1,221 +0,0 @@ -import ctypes - -from ctree.types import CtreeType, CtreeTypeResolver, TypeFetcher, get_ctree_type - - -class CType(CtreeType): - """Base class for all built-in CTypes.""" - - def codegen(self): - from ctree.c.codegen import CCodeGen - - return CCodeGen().visit(self) - - def as_ctype(self): - return self._ctype - - -class Void(CType): - _ctype = ctypes.c_void_p - - -class Char(CType): - _ctype = ctypes.c_char - - -class UChar(CType): - _ctype = ctypes.c_ubyte - - -class Short(CType): - _ctype = ctypes.c_short - - -class UShort(CType): - _ctype = ctypes.c_ushort - - -class Int(CType): - _ctype = ctypes.c_int - - -class UInt(CType): - _ctype = ctypes.c_uint - - -class Long(CType): - _ctype = ctypes.c_long - - -class ULong(CType): - _ctype = ctypes.c_ulong - - -class Float(CType): - _ctype = ctypes.c_float - - -class Double(CType): - _ctype = ctypes.c_double - - -class LongDouble(CType): - _ctype = ctypes.c_longdouble - -class Ptr(CType): - """ - Pointer type. - """ - _fields = ['base_type'] - - def __init__(self, base_type=None): - self.base_type = base_type if base_type else Void() - - def as_ctype(self): - return ctypes.POINTER(self.base_type.as_ctype()) - - -class FuncType(CType): - _fields = ['return_type', 'arg_types'] - - def __init__(self, return_type=Void(), arg_types=None): - self.return_type = return_type - self.arg_types = arg_types if arg_types else [] - - def as_ctype(self): - return_ctype = self.return_type.as_ctype() - arg_ctypes = [arg_type.as_ctype() for arg_type in self.arg_types] - return ctypes.CFUNCTYPE(return_ctype, *arg_ctypes) - - -class NdPointer(CType): - def __init__(self, dtype=None, ndim=1, shape=1, flags=None): - from numpy.ctypeslib import ndpointer - - self.ptr = ndpointer(dtype, ndim, shape, flags) - - def get_base_type(self): - return get_ctree_type(self.ptr._dtype_) - - def as_ctype(self): - return self.ptr - - @staticmethod - def to(ndarray): - """Factory routine for creating an NdPointer to an existing array.""" - return NdPointer(ndarray.dtype, ndarray.ndim, - ndarray.shape, ndarray.flags) - - -class FILE(CType): - pass - - -class CTypeResolver(CtreeTypeResolver): - @staticmethod - def resolve(obj): - if isinstance(obj, int): - return Long() - elif isinstance(obj, float): - return Double() - elif isinstance(obj, str): - return Char() if len(obj) == 1 else Ptr(Char()) - - -class NumpyTypeResolver(CtreeTypeResolver): - @staticmethod - def resolve(ty): - import numpy as np - - # pylint: disable=no-member - if ty == np.int32: - return Int() - elif ty == np.int64: - return Long() - elif ty == np.float32: - return Float() - elif ty == np.float64: - return Double() - # pylint: enable=no-member - - -class CTypeFetcher(TypeFetcher): - """ - Dynamically computes the type of the Expression. - """ - - def visit_String(self, node): - return Ptr(Char()) - - def visit_SymbolRef(self, node): - if node.type is not None: - return node.type - else: - #decl = DeclFinder().find(node) - #return decl.get_type() - return "??" - - def visit_Constant(self, node): - return get_ctree_type(node.value) - - def visit_BinaryOp(self, node): - from ctree.c.nodes import Op - - lhs = node.left.get_type() - rhs = node.right.get_type() - if isinstance(node.op, (Op.Add, Op.Sub, Op.Mul, Op.Div, Op.Mod, - Op.BitAnd, Op.BitOr, Op.BitXor, - Op.BitShL, Op.BitShR)): - return self._usual_arithmetic_convert(lhs, rhs) - elif isinstance(node.op, (Op.Lt, Op.Gt, Op.LtE, Op.GtE, Op.Eq, Op.NotEq, - Op.And, Op.Or)): - return Int() - elif isinstance(node.op, Op.Comma): - return rhs - elif isinstance(node.op, Op.ArrayRef): - return lhs.base - else: - raise Exception("Cannot determine return type of (%s %s %s)." % (lhs, node.op, rhs)) - - @staticmethod - def _usual_arithmetic_convert(t0, t1): - """ - Computes the return type of an arithmetic operator applied to arguments of - the built-in numeric types. - See C89 6.2.5.1. - """ - - if t0 == LongDouble() or t1 == LongDouble(): - return LongDouble() - elif t0 == Double() or t1 == Double(): - return Double() - elif t0 == Float() or t1 == Float(): - return Float() - else: - t0 = CTypeFetcher._integer_promote(t0) - t1 = CTypeFetcher._integer_promote(t1) - if t0 == ULong() or t1 == ULong(): - return ULong() - elif t0 == Long() or t1 == Long(): - return Long() - elif t0 == UInt() or t1 == UInt(): - return UInt() - elif t0 == Int() or t1 == Int(): - return Int() - else: - raise Exception("Failed to apply usual arith conversion (c89 6.2.1.5) to types: %s, %s." % - (t0, t1)) - - @staticmethod - def _integer_promote(t): - """ - Promote small types to integers accd to c89 6.2.1.1. - """ - if isinstance(t, (Int, UInt, Long, ULong)): - return t - elif isinstance(t, (Char, UChar, Short, UShort)): - return Int() - else: - raise Exception("Cannot promote type %s to an integer-type." % t) - - diff --git a/ctree/cilk/dotgen.py b/ctree/cilk/dotgen.py index 0c8696d..af1d172 100644 --- a/ctree/cilk/dotgen.py +++ b/ctree/cilk/dotgen.py @@ -1,15 +1,14 @@ """ -DOT generation for Cilk. +DOT labeller for Cilk. """ -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenLabeller # --------------------------------------------------------------------------- -# DOT generator +# DOT labeller - -class CilkDotGen(DotGenVisitor): +class CilkDotLabeller(DotGenLabeller): """ - Visitor to generator DOT. + Visitor to label DOT nodes. """ pass diff --git a/ctree/cilk/nodes.py b/ctree/cilk/nodes.py index 6268a9e..02a4d85 100644 --- a/ctree/cilk/nodes.py +++ b/ctree/cilk/nodes.py @@ -14,6 +14,6 @@ def codegen(self, indent=0): return CilkCodeGen(indent).visit(self) def _to_dot(self, _): - from ctree.cilk.dotgen import CilkDotGen + from ctree.cilk.dotgen import CilkDotLabeller - return CilkDotGen().visit(self) + return CilkDotLabeller().visit(self) diff --git a/ctree/codegen.py b/ctree/codegen.py index d2ff2aa..b78cacd 100644 --- a/ctree/codegen.py +++ b/ctree/codegen.py @@ -1,9 +1,12 @@ +from __future__ import print_function + """ base class for generating code appropriate to the selected backend """ from ctree.visitors import NodeVisitor from ctree.util import flatten + class CodeGenVisitor(NodeVisitor): """ Return a string containing the program text. @@ -19,14 +22,20 @@ def _tab(self): """return correct spaces if tab found""" return " " * self._indent - def _genblock(self, forest, insert_curly_brackets=True, increase_indent=True): + def _genblock(self, forest, insert_curly_brackets=True, + increase_indent=True): """generate block of code adding semi colons as necessary""" if increase_indent: self._indent += 1 body = "" for tree in flatten(forest): - semicolon_opt = ";" if tree._requires_semicolon() else "" - body += self._tab() + tree.codegen(self._indent) + semicolon_opt + "\n" + if not hasattr(tree, '_requires_semicolon'): + body += self._tab() + str(tree) + "\n" + else: + semicolon_opt = ";" if tree._requires_semicolon() else "" + block = tree.codegen(self._indent) + if block is not "": + body += self._tab() + block + semicolon_opt + "\n" if increase_indent: self._indent -= 1 if insert_curly_brackets: @@ -34,10 +43,18 @@ def _genblock(self, forest, insert_curly_brackets=True, increase_indent=True): else: return "\n%s" % body - def _parentheses(self, node): + def _parenthesize(self, parent, child): """A format string that includes parentheses if needed.""" - return "(%s)" if self._requires_parentheses(node) else "%s" + try: + if self._requires_parentheses(parent, child) or \ + child._force_parentheses is True: + return "(%s)" % child + else: + return "%s" % child + except AttributeError: + print("{} {} has no attribute _force_parentheses".format(type(child), child)) + raise - def _requires_parentheses(self, _): - """TODO: figure out why this is always true""" + def _requires_parentheses(self, parent, child): + """True by default.""" return True diff --git a/ctree/cpp/codegen.py b/ctree/cpp/codegen.py index e132537..858f3fc 100644 --- a/ctree/cpp/codegen.py +++ b/ctree/cpp/codegen.py @@ -16,8 +16,9 @@ def visit_CppInclude(self, node): else: return '#include "%s"' % node.target - def visit_Comment(self, node): - return "// %s" % node.text + def visit_CppComment(self, node): + return "// " + ("\n" + self._tab() + "// ").join( + node.text.splitlines()) def visit_CppDefine(self, node): params = ", ".join(map(str, node.params)) diff --git a/ctree/cpp/dotgen.py b/ctree/cpp/dotgen.py index 9f5f36e..89bcb0b 100644 --- a/ctree/cpp/dotgen.py +++ b/ctree/cpp/dotgen.py @@ -2,19 +2,19 @@ DOT generation for C preprocessor directives. """ -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenLabeller -class CppDotGen(DotGenVisitor): +class CppDotLabeller(DotGenLabeller): """ Visitor to generator DOT. """ - def label_CppInclude(self, node): + def visit_CppInclude(self, node): if node.angled_brackets: return "target: <%s>" % node.target else: return 'target: "%s"' % node.target - def label_Comment(self, node): - return node.text.replace('"', r"\"") + def visit_CppComment(self, node): + return "// " + node.text.replace('"', r"\"") diff --git a/ctree/cpp/nodes.py b/ctree/cpp/nodes.py index 8f41dea..f7f1ece 100644 --- a/ctree/cpp/nodes.py +++ b/ctree/cpp/nodes.py @@ -13,10 +13,10 @@ def codegen(self, indent=0): return CppCodeGen(indent).visit(self) - def _to_dot(self): - from ctree.cpp.dotgen import CppDotGen + def label(self): + from ctree.cpp.dotgen import CppDotLabeller - return CppDotGen().visit(self) + return CppDotLabeller().visit(self) def _requires_semicolon(self): return False @@ -29,15 +29,19 @@ def __init__(self, target="", angled_brackets=True): self.target = target self.angled_brackets = angled_brackets + def __hash__(self): + return hash(self.target) ^ hash(self.angled_brackets) -class Comment(CppNode): + +class CppComment(CppNode): """Represents // foo""" def __init__(self, text=""): - assert "\n" not in text, "Comment only supports single-line comments." self.text = text + class CppDefine(CppNode): + _fields = ['name', 'params', 'body'] def __init__(self, name=None, params=None, body=None): self.name = name diff --git a/ctree/defaults.cfg b/ctree/defaults.cfg index 1361d69..3bd032d 100644 --- a/ctree/defaults.cfg +++ b/ctree/defaults.cfg @@ -1,7 +1,24 @@ [jit] -CC = clang -CFLAGS = -O2 -PRESERVE_SRC_DIR = False +COMPILE_PATH = ./compiled +CACHE = False + +[c] +CC = gcc +CFLAGS = -fPIC -O2 -std=c99 +LDFLAGS = + +[omp] +CC = gcc +CFLAGS = -fPIC -std=c99 -O2 -I/opt/intel/composerxe/include -fopenmp +LDFLAGS = + +[opencl] +CC = gcc +CFLAGS = -fPIC -std=c99 -O2 +# For Linux +# LDFLAGS = -lOpenCL +# For OSX +LDFLAGS = -framework OpenCL [log] # maximum number of lines to show when programs are printed to the log diff --git a/ctree/dotgen.py b/ctree/dotgen.py index 700d3b6..79ba0bf 100644 --- a/ctree/dotgen.py +++ b/ctree/dotgen.py @@ -4,14 +4,44 @@ from ctree.util import enumerate_flatten +def label_for_py_ast_nodes(self): + from ctree.py.dotgen import PyDotLabeller + + return PyDotLabeller().visit(self) + +def to_dot_outer_for_py_ast_nodes(self): + return "digraph mytree {\n%s}" % self._to_dot() + +def to_dot_inner_for_py_ast_nodes(self): + from ctree.dotgen import DotGenVisitor + + return DotGenVisitor().visit(self) + +""" +Bind to_dot_for_py_ast_nodes to all classes that derive from ast.AST. Ideally +we'd be able to bind one method to ast.AST, but it's a built-in type so we +can't. +""" +for entry in ast.__dict__.values(): + try: + if issubclass(entry, ast.AST): + entry.label = label_for_py_ast_nodes + entry.to_dot = to_dot_outer_for_py_ast_nodes + entry._to_dot = to_dot_inner_for_py_ast_nodes + except TypeError: + pass + + +class DotGenLabeller(NodeVisitor): + def generic_visit(self, node): + return "" + + class DotGenVisitor(NodeVisitor): """ Generates a representation of the AST in the DOT graph language. See http://en.wikipedia.org/wiki/DOT_(graph_description_language) - - We can use pydot to do this, instead of using plain string concatenation. """ - @staticmethod def _qualified_name(obj): """ @@ -22,52 +52,20 @@ def _qualified_name(obj): def label(self, node): """ A string to provide useful information for visualization, debugging, etc. - This routine will attempt to call a label_XXX routine for class XXX, if - such a routine exists (much like the visit_XXX routines). """ - out_string = r"%s\n" % type(node).__name__ - labeller = getattr(self, "label_" + type(node).__name__, None) - if labeller: - out_string += labeller(node) - return out_string + return r"%s\n%s" % (type(node).__name__, node.label()) def generic_visit(self, node): + # label this node out_string = 'n%s [label="%s"];\n' % (id(node), self.label(node)) - # edge to parent - if hasattr(node, 'parent') and node.parent is not None: - out_string += 'n%s -> n%s [label="parent",style=dotted];\n' % (id(node), id(node.parent)) - # edges to children for fieldname, fieldvalue in ast.iter_fields(node): for index, child in enumerate_flatten(fieldvalue): if isinstance(child, ast.AST): suffix = "".join(["[%d]" % i for i in index]) - out_string += 'n%d -> n%d [label="%s%s"];\n' % (id(node), id(child), fieldname, suffix) - out_string += _to_dot(child) + out_string += 'n{} -> n{} [label="{}{}"];\n'.format( + id(node), id(child), fieldname, suffix) + out_string += self.visit(child) return out_string - - -def _to_dot(node): - """ - Convert node to DOT, even if it's a Python AST node. - """ - from ctree.nodes import CtreeNode - from ctree.py.dotgen import PyDotGen - - assert isinstance(node, ast.AST), \ - "Cannot convert %s to DOT." % type(node) - if isinstance(node, CtreeNode): - return node._to_dot() - else: - return PyDotGen().visit(node) - - -def to_dot(node): - """ - Returns a DOT representation of 'node' suitable for viewing with a DOT viewer like Graphviz. - """ - assert isinstance(node, ast.AST), \ - "Cannot convert %s to DOT." % type(node) - return "digraph myprogram {\n%s}" % _to_dot(node) diff --git a/ctree/frontend.py b/ctree/frontend.py index 9d09dd7..7d0a006 100644 --- a/ctree/frontend.py +++ b/ctree/frontend.py @@ -14,3 +14,93 @@ def get_ast(obj): indented_program_txt = inspect.getsource(obj) program_txt = textwrap.dedent(indented_program_txt) return ast.parse(program_txt) + + +""" +A pretty-printing dump function for the ast module. The code was copied from +the ast.dump function and modified slightly to pretty-print. + +Alex Leone (acleone ~AT~ gmail.com), 2010-01-30 + +From http://alexleone.blogspot.co.uk/2010/01/python-ast-pretty-printer.html +""" + +from ast import * + +def dump(node, annotate_fields=True, include_attributes=False, indent=' '): + """ + Return a formatted dump of the tree in *node*. This is mainly useful for + debugging purposes. The returned string will show the names and the values + for fields. This makes the code impossible to evaluate, so if evaluation is + wanted *annotate_fields* must be set to False. Attributes such as line + numbers and column offsets are not dumped by default. If this is wanted, + *include_attributes* can be set to True. + """ + def _format(node, level=0): + if isinstance(node, AST): + fields = [(a, _format(b, level)) for a, b in iter_fields(node)] + if include_attributes and node._attributes: + fields.extend([(a, _format(getattr(node, a), level)) + for a in node._attributes]) + return ''.join([ + node.__class__.__name__, + '(', + ', '.join(('%s=%s' % field for field in fields) + if annotate_fields else + (b for a, b in fields)), + ')']) + elif isinstance(node, list): + lines = ['['] + lines.extend((indent * (level + 2) + _format(x, level + 2) + ',' + for x in node)) + if len(lines) > 1: + lines.append(indent * (level + 1) + ']') + else: + lines[-1] += ']' + return '\n'.join(lines) + return repr(node) + + if not isinstance(node, AST): + raise TypeError('expected AST, got %r' % node.__class__.__name__) + return _format(node) + +def parseprint(code, filename="", mode="exec", **kwargs): + """Parse some code from a string and pretty-print it.""" + node = parse(code, mode=mode) # An ode to the code + print(dump(node, **kwargs)) + +# Short name: pdp = parse, dump, print +pdp = parseprint + +def load_ipython_extension(ip): # pragma: no cover + from IPython.core.magic import Magics, magics_class, cell_magic + from IPython.core import magic_arguments + + @magics_class + class AstMagics(Magics): + + @magic_arguments.magic_arguments() + @magic_arguments.argument( + '-m', '--mode', default='exec', + help="The mode in which to parse the code. Can be exec (the default), " + "eval or single." + ) + @cell_magic + def dump_ast(self, line, cell): + """Parse the code in the cell, and pretty-print the AST.""" + args = magic_arguments.parse_argstring(self.dump_ast, line) + parseprint(cell, mode=args.mode) + + ip.register_magics(AstMagics) + +if __name__ == '__main__': # pragma: no cover + import sys, tokenize + for filename in sys.argv[1:]: + print('=' * 50) + print('AST tree for', filename) + print('=' * 50) + with tokenize.open(filename) as f: + fstr = f.read() + + parseprint(fstr, filename=filename, include_attributes=True) + print() diff --git a/ctree/jit.py b/ctree/jit.py index 5abaa0e..4f08726 100644 --- a/ctree/jit.py +++ b/ctree/jit.py @@ -1,18 +1,46 @@ -"""just in time utilities""" +""" +Just-in-time compilation support. +""" + +import abc import copy -import shutil +import os +import re +import ast +import logging +import inspect +import hashlib +import json +from collections import namedtuple import tempfile + import ctree from ctree.nodes import Project from ctree.analyses import VerifyOnlyCtreeNodes -from ctree.util import highlight +from ctree.frontend import get_ast, dump +from ctree.transforms import DeclarationFiller +from ctree.c.nodes import CFile, MultiNode +if ctree.OCL_ENABLED: + from ctree.ocl.nodes import OclFile +from ctree.nodes import File -import llvm.core as ll +log = logging.getLogger(__name__) -import logging -log = logging.getLogger(__name__) +def getFile(filepath): + """ + Takes a filepath and returns a specialized File instance (i.e. OclFile, + CFile, etc) + """ + file_types = [CFile] + if ctree.OCL_ENABLED: + file_types.append(OclFile) + ext_map = {'.'+t._ext: t for t in file_types} + path, filename = os.path.split(filepath) + name, ext = os.path.splitext(filename) + filetype = ext_map[ext] + return filetype(name=name, path=path) class JitModule(object): @@ -21,21 +49,24 @@ class JitModule(object): """ def __init__(self): - self.compilation_dir = tempfile.mkdtemp(prefix="ctree-", - dir=tempfile.gettempdir()) - self.ll_module = ll.Module.new('ctree') + import os + if ctree.CONFIG.get('jit', 'COMPILE_PATH') and ctree.CONFIG.getboolean('jit', 'CACHE'): + ctree_dir = ctree.CONFIG.get('jit','COMPILE_PATH') + # write files to $TEMPDIR/ctree/run-XXXX + else: + ctree_dir = os.path.join(tempfile.gettempdir(), "ctree") + if not os.path.exists(ctree_dir): + os.mkdir(ctree_dir) + #self.compilation_dir = tempfile.mkdtemp(prefix="run-", dir=ctree_dir) + self.ll_module = None self.exec_engine = None - log.info("temporary compilation directory is: %s", - self.compilation_dir) - - def __del__(self): - if not ctree.CONFIG.get("jit", "PRESERVE_SRC_DIR"): - log.info("removing temporary compilation directory %s.", - self.compilation_dir) - shutil.rmtree(self.compilation_dir) def _link_in(self, submodule): - self.ll_module.link_in(submodule) + self.so_file_name = submodule + # if self.ll_module is not None: + # self.ll_module.link_in(submodule) + # else: + # self.ll_module = submodule def get_callable(self, entry_point_name, entry_point_typesig): """ @@ -43,45 +74,51 @@ def get_callable(self, entry_point_name, entry_point_typesig): """ # get llvm represetation of function - ll_function = self.ll_module.get_function_named(entry_point_name) + # ll_function = self.ll_module.get_function(entry_point_name) + import ctypes + lib = ctypes.cdll.LoadLibrary(self.so_file_name) + func_ptr = getattr(lib, entry_point_name) + func_ptr.argtypes = entry_point_typesig._argtypes_ + func_ptr.restype = entry_point_typesig._restype_ + # func = func_ptr # run jit compiler - from llvm.ee import EngineBuilder + # from llvm.ee import EngineBuilder + # self.exec_engine = llvm.create_jit_compiler(self.ll_module) - self.exec_engine = \ - EngineBuilder.new(self.ll_module).mcjit(True).opt(3).create() - - c_func_ptr = self.exec_engine.get_pointer_to_function(ll_function) + # c_func_ptr = self.exec_engine.get_pointer_to_global(ll_function) # cast c_func_ptr to python callable using ctypes - return entry_point_typesig(c_func_ptr) + return func_ptr -class _ConcreteSpecializedFunction(object): +class ConcreteSpecializedFunction(object): """ A function backed by generated code. """ + __metaclass__ = abc.ABCMeta - def __init__(self, entry_point_name, project, entry_point_typesig, extra_args=tuple()): - assert isinstance(project, Project), \ - "Expected a Project but it got a %s." % type(project) - assert project.parent is None, \ - "Expected null project.parent, but got: %s." % type(project.parent) + def _compile(self, entry_point_name, project_node, entry_point_typesig, + **kwargs): + """ + Returns a python callable. + """ + assert isinstance(project_node, Project), \ + "Expected a Project but it got a %s." % type(project_node) - VerifyOnlyCtreeNodes().visit(project) + VerifyOnlyCtreeNodes().visit(project_node) - self.module = project.codegen() - highlighted = highlight(str(self.module.ll_module), 'llvm') - log.debug("full LLVM program is: <<<\n%s\n>>>" % highlighted) - self.fn = self.module.get_callable(entry_point_name, - entry_point_typesig) - self._extra_args = extra_args + self._module = project_node.codegen(**kwargs) - def __call__(self, *args, **kwargs): - assert not kwargs, \ - "Passing kwargs to SpecializedFunction.__call__ isn't supported." + # if log.getEffectiveLevel() == 'debug': + # highlighted = highlight(str(self._module.ll_module), 'llvm') + # log.debug("full LLVM program is: <<<\n%s\n>>>" % highlighted) - return self.fn(*(args + self._extra_args), **kwargs) + return self._module.get_callable(entry_point_name, entry_point_typesig) + + @abc.abstractmethod + def __call__(self, *args, **kwargs): + pass class LazySpecializedFunction(object): @@ -90,58 +127,264 @@ class LazySpecializedFunction(object): code just-in-time. """ - def __init__(self, py_ast, entry_point_name): - self.original_tree = py_ast - self.entry_point_name = entry_point_name + ProgramConfig = namedtuple('ProgramConfig', + ['args_subconfig', 'tuner_subconfig']) + _directory_fields = ['__class__.__name__', 'backend_name'] + + class NameExtractor(ast.NodeVisitor): + """ + Extracts the first functiondef name found + """ + def visit_FunctionDef(self, node): + return node.name + + def generic_visit(self, node): + for field, value in ast.iter_fields(node): + if isinstance(value, list): + for item in value: + if isinstance(item, ast.AST): + res = self.visit(item) + if res: + return res + elif isinstance(value, ast.AST): + res = self.visit(value) + if res: + return res + + def __init__(self, py_ast=None, sub_dir=None, backend_name="default"): + self._hash_cache = None + if py_ast is not None and \ + self.apply is not LazySpecializedFunction.apply: + raise TypeError('Cannot define apply and pass py_ast') + self.original_tree = py_ast or \ + (get_ast(self.apply) + if self.apply is not LazySpecializedFunction.apply else None) self.concrete_functions = {} # config -> callable map self._tuner = self.get_tuning_driver() + self.sub_dir = sub_dir or \ + self.NameExtractor().visit(self.original_tree) or \ + hex(hash(self))[2:] + self.backend_name = backend_name + + + @property + def original_tree(self): + return copy.deepcopy(self._original_tree) + + @property + def tree(self): + return self._original_tree + + @original_tree.setter + def original_tree(self, value): + if not hasattr(self, '_original_tree'): + self._original_tree = value + elif ast.dump(self.__original_tree, True, True) != \ + ast.dump(value, True, True): + raise AttributeError('Cannot redefine the ast') + + @property + def info_filename(self): + return 'info.json' + + def get_info(self, path): + info_filepath = os.path.join(path, self.info_filename) + if not os.path.exists(info_filepath): + return {'hash': None, 'files': []} + with open(info_filepath) as info_file: + return json.load(info_file) + + def set_info(self, path, dictionary): + info_filepath = os.path.join(path, self.info_filename) + with open(info_filepath, 'w') as info_file: + return json.dump(dictionary, info_file) @staticmethod - def _hash_dict(o): - if isinstance(o, dict): - return hash(frozenset(o.items())) + def _hash(o): + if isinstance(o, dict) and type(o).__hash__ is dict.__hash__: + return hash(frozenset( + LazySpecializedFunction._hash(item) for item in o.items() + )) else: - return hash(o) + try: + return hash(o) + except TypeError: + return hash(str(o)) + + def __hash__(self): + # mro = type(self).mro() + # result = hashlib.sha512(''.encode()) + # for klass in mro: + # if issubclass(klass, LazySpecializedFunction): + # try: + # result.update(inspect.getsource(klass).encode()) + # except IOError: + # # means source can't be found. Well, can't do anything + # # about that I don't think + # pass + # else: + # pass + # if self.original_tree is not None: + # tree_str = ast.dump(self.original_tree, + # annotate_fields=True, include_attributes=True) + # result.update(tree_str.encode()) + # return int(result.hexdigest(), 16) + if self._hash_cache is not None: + return self._hash_cache + try: + self_hash = hash(inspect.getsource(type(self)).encode()) + except TypeError: + self_hash = 1 + #self_hash = 1 + tree_hash = hash(dump(self._original_tree, annotate_fields=True, include_attributes=True)) + self._hash_cache = self_hash * tree_hash + return self._hash_cache + + def config_to_dirname(self, program_config): + """Returns the subdirectory name under .compiled/funcname""" + # fixes the directory names and squishes invalid chars + regex_filter = re.compile(r"""[/\?%*:|"<>()'{} -]""") + + def deep_getattr(obj, s): + parts = s.split('.') + for part in parts: + obj = getattr(obj, part) + return obj + + path_parts = [ + self.sub_dir, + str(type(self)), + str(self._hash(program_config.args_subconfig)), + str(self._hash(program_config.tuner_subconfig)) + ] + for attrib in self._directory_fields: + path_parts.append(str(deep_getattr(self, attrib))) + filtered_parts = [ + str(re.sub(regex_filter, '_', part)) for part in path_parts] + compile_path = str(ctree.CONFIG.get('jit', 'COMPILE_PATH')) + + path = os.path.join(compile_path, *filtered_parts) + return re.sub('_+', '_', path) + + def get_program_config(self, args, kwargs): + # Don't break old specializers that don't support kwargs + try: + args_subconfig = self.args_to_subconfig(args, kwargs) + except TypeError: + args_subconfig = self.args_to_subconfig(args) + try: + self._tuner.configs.send((args, args_subconfig)) + except TypeError: + "Can't send into an unstarted generator" + pass + tuner_subconfig = next(self._tuner.configs) + log.info("tuner subconfig: %s", tuner_subconfig) + log.info("arguments subconfig: %s", args_subconfig) + + return self.ProgramConfig(args_subconfig, tuner_subconfig) + + def get_transform_result(self, program_config, dir_name, cache=True): + info = self.get_info(dir_name) + # check to see if the necessary code is in the persistent cache + if hash(self) != info['hash'] and self.original_tree is not None \ + or not cache: + # need to run transform() for code generation + log.info('Hash miss. Running Transform') + ctree.STATS.log("Filesystem cache miss") + transform_result = self.run_transform(program_config) + + # Saving files to cache directory + for source_file in transform_result: + assert isinstance(source_file, File), \ + "Transform must return an iterable of Files" + source_file.path = dir_name + + new_info = {'hash': hash(self), + 'files': [os.path.join(f.path, f.get_filename()) + for f in transform_result]} + self.set_info(dir_name, new_info) + + else: + log.info('Hash hit. Skipping transform') + ctree.STATS.log('Filesystem cache hit') + files = [getFile(path) for path in info['files']] + transform_result = files + return transform_result def __call__(self, *args, **kwargs): """ Determines the program_configuration to be run. If it has yet to be - built, build it. Then, execute it. + built, build it. Then, execute it. If the selected + program_configuration for this function has already been code + generated for, this method draws from the cache. """ ctree.STATS.log("specialized function call") - assert not kwargs, \ - "Passing kwargs to specialized functions isn't supported." log.info("detected specialized function call with arg types: %s", - [type(a) for a in args]) + [type(a) for a in args] + + [type(kwargs[key]) for key in kwargs]) - args_subconfig = self.args_to_subconfig(args) - tuner_subconfig = next(self._tuner.configs) - program_config = (args_subconfig, tuner_subconfig) + program_config = self.get_program_config(args, kwargs) + dir_name = self.config_to_dirname(program_config) - log.info("tuner subconfig: %s", tuner_subconfig) - log.info("arguments subconfig: %s", args_subconfig) + if not os.path.exists(dir_name): + os.makedirs(dir_name) - config_hash = hash((self._hash_dict(args_subconfig), - self._hash_dict(tuner_subconfig))) + config_hash = dir_name - if config_hash in self.concrete_functions: + # checks to see if the necessary code is in the run-time cache + if ctree.CONFIG.getboolean('jit', 'CACHE') and \ + config_hash in self.concrete_functions: ctree.STATS.log("specialized function cache hit") log.info("specialized function cache hit!") + csf = self.concrete_functions[config_hash] + else: ctree.STATS.log("specialized function cache miss") log.info("specialized function cache miss.") - translator_result = self.transform( - copy.deepcopy(self.original_tree), - program_config - ) - - self.concrete_functions[config_hash] = _ConcreteSpecializedFunction( - self.entry_point_name, - *translator_result - ) - - return self.concrete_functions[config_hash](*args) + transform_result = self.get_transform_result( + program_config, dir_name) + + csf = self.finalize(transform_result, program_config) + assert isinstance(csf, ConcreteSpecializedFunction), \ + "Expected a ctree.jit.ConcreteSpecializedFunction, \ + but got a %s." % type(csf) + self.concrete_functions[config_hash] = csf + + return csf(*args, **kwargs) + + def run_transform(self, program_config): + transform_result = self.transform( + self.original_tree, + program_config + ) + if not isinstance(transform_result, (tuple, list)): + transform_result = (transform_result,) + + transform_result = [DeclarationFiller().visit(source_file) + if isinstance(source_file, CFile) else source_file + for source_file in transform_result] + return transform_result + + @classmethod + def from_function(cls, func, folder_name=''): + class Replacer(ast.NodeTransformer): + def visit_Module(self, node): + return MultiNode(body=[self.visit(i) for i in node.body]) + + def visit_FunctionDef(self, node): + if node.name == func.__name__: + node.name = 'apply' + node.body = [self.visit(item) for item in node.body] + return node + + def visit_Name(self, node): + if node.id == func.__name__: + node.id = 'apply' + return node + + func_ast = Replacer().visit(get_ast(func)) + return cls(py_ast=func_ast, sub_dir=folder_name or func.__name__) def report(self, *args, **kwargs): """ @@ -159,13 +402,20 @@ def transform(self, tree, program_config): """ raise NotImplementedError() + def finalize(self, transform_result, program_config): + """ + This function will be passed the result of transform. The specializer + should return an ConcreteSpecializedFunction. + """ + raise NotImplementedError("Finalize must be implemented") + def get_tuning_driver(self): """ Define the space of possible implementations. """ - from ctree.tune import NullTuningDriver + from ctree.tune import ConstantTuningDriver - return NullTuningDriver() + return ConstantTuningDriver('') def args_to_subconfig(self, args): """ @@ -176,4 +426,8 @@ def args_to_subconfig(self, args): log.warn("arguments will not influence program_config. " + "Consider overriding args_to_subconfig() in %s.", type(self).__name__) - return dict() + return {} + + @staticmethod + def apply(*args): + raise NotImplementedError() diff --git a/ctree/metrics/watts_up_reader.py b/ctree/metrics/watts_up_reader.py index af302fd..72370ad 100644 --- a/ctree/metrics/watts_up_reader.py +++ b/ctree/metrics/watts_up_reader.py @@ -42,7 +42,6 @@ import select import collections import argparse -import sys class WattsUpReader(object): @@ -57,7 +56,7 @@ class WattsUpReader(object): ) def __init__(self, port_name=None, verbose=False): - if port_name == None: + if port_name is None: from ctree import CONFIG port_name = CONFIG.get('wattsup', 'port') self.port_name = port_name @@ -295,8 +294,6 @@ def guess_port(): if __name__ == "__main__": - # default port name is based on right hand side usb on macbookpro - # usb_port_name = "/dev/tty.usbserial-A600KI7M" if len(sys.argv) < 2 else sys.argv[1] parser = argparse.ArgumentParser(description="interface to WattsUpPro usb power meter") parser.add_argument( '-i', '--interactive', help='interactive mode, allows direct communcation with device', action="store_true" diff --git a/ctree/nodes.py b/ctree/nodes.py index e6b39bc..cb6038e 100644 --- a/ctree/nodes.py +++ b/ctree/nodes.py @@ -3,14 +3,18 @@ """ import logging +import os.path log = logging.getLogger(__name__) import ast +import collections from ctree.codegen import CodeGenVisitor -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenVisitor, DotGenLabeller from ctree.util import flatten +import ctree +import os class CtreeNode(ast.AST): @@ -20,15 +24,8 @@ class CtreeNode(ast.AST): def __init__(self): """Initialize a new AST Node.""" super(CtreeNode, self).__init__() - self.parent = None - - def __setattr__(self, name, value): - """Set attribute and preserve parent pointers.""" - if name != "parent": - for child in flatten(value): - if isinstance(child, CtreeNode): - child.parent = self - super(CtreeNode, self).__setattr__(name, value) + self.deleted = False + self._force_parentheses = False def __str__(self): return self.codegen() @@ -36,24 +33,25 @@ def __str__(self): def codegen(self, indent=0): raise Exception("Node class %s should override codegen()" % type(self)) + def delete(self): + self.codegen = self.no_code_gen + self.deleted = True + + def no_code_gen(self, *args): + return "" + + def to_dot(self): + """Retrieve the AST in DOT format for vizualization.""" + return "digraph mytree {\n%s}" % self._to_dot() + def _to_dot(self): """Retrieve the AST in DOT format for vizualization.""" - raise Exception("Node class %s should override _to_dot()" % type(self)) + return DotGenVisitor().visit(self) def _requires_semicolon(self): """When coverted to a string, this node should be followed by a semicolon.""" return True - def get_root(self): - """ - Traverse the parent pointer list to find the eldest - parent without a parent, aka the root. - """ - root = self - while root.parent is not None: - root = root.parent - return root - def find_all(self, node_class, **kwargs): """ Returns a generator that yields all nodes of type @@ -64,7 +62,7 @@ def find_all(self, node_class, **kwargs): """ def pred(node): - if type(node) == node_class: + if isinstance(node, node_class): for attr, value in kwargs.items(): try: if getattr(node, attr) != value: @@ -99,44 +97,16 @@ def find_if(self, pred): if pred(node): yield node - def replace(self, new_node): - """ - Replace the current node with 'new_node'. - """ - parent = self.parent - assert self.parent, "Tried to replace a node without a parent." - for fieldname, child in ast.iter_fields(parent): - if child is self: - setattr(parent, fieldname, new_node) - elif isinstance(child, list) and self in child: - child[child.index(self)] = new_node - return new_node - - def insert_before(self, older_sibling): - """ - Insert the given node just before 'self' in the current scope. Requires - that 'self' be contained in a list. - """ - parent = self.parent - assert self.parent, "Tried to insert_before a node without a parent." - for fieldname, child in ast.iter_fields(parent): - if isinstance(child, list) and self in child: - child.insert(child.index(self), older_sibling) - return - raise Exception("Couldn't perform insertion.") - - def insert_after(self, younger_sibling): - """ - Insert the given node just before 'self' in the current scope. Requires - that 'self' be contained in a list. - """ - parent = self.parent - assert self.parent, "Tried to insert_before a node without a parent." - for fieldname, child in ast.iter_fields(parent): - if isinstance(child, list) and self in child: - child.insert(child.index(self) + 1, younger_sibling) - return - raise Exception("Couldn't perform insertion.") + def lift(self, **kwargs): + for key, val in kwargs.items(): + attr = "_lift_%s" % key + setattr(self, attr, getattr(self, attr, []) + val) + type(self)._fields.append(attr) + + def __eq__(self, other): + """Two nodes are equal if their attributes are equal.""" + return self.__dict__ == getattr(other, '__dict__', None) + # --------------------------------------------------------------------------- @@ -148,7 +118,7 @@ class CommonNode(CtreeNode): def codegen(self, indent=0): return CommonCodeGen(indent).visit(self) - def _to_dot(self): + def label(self): return CommonDotGen().visit(self) @@ -156,9 +126,11 @@ class Project(CommonNode): """Holds a list files.""" _fields = ['files'] - def __init__(self, files=None): + def __init__(self, files=None, indent=0, compilation_dir = ''): self.files = files if files else [] super(Project, self).__init__() + self.compilation_dir = compilation_dir + self.indent = indent def codegen(self, indent=0): """ @@ -167,36 +139,84 @@ def codegen(self, indent=0): """ from ctree.jit import JitModule - module = JitModule() + self._module = JitModule() # now that we have a concrete compilation dir, resolve references to it from ctree.transformations import ResolveGeneratedPathRefs - - resolver = ResolveGeneratedPathRefs(module.compilation_dir) - self.files = [resolver.visit(f) for f in self.files] - log.info("automatically resolved %d GeneratedPathRef node(s).", resolver.count) + # + # resolver = ResolveGeneratedPathRefs(self._module.compilation_dir) + # self.files = [resolver.visit(f) for f in self.files] + # if resolver.count: + # log.info("automatically resolved %d GeneratedPathRef node(s).", resolver.count) # transform all files into llvm modules and link them into the master module for f in self.files: - submodule = f._compile(f.codegen(), module.compilation_dir) + submodule = f._compile(f.codegen()) if submodule: - module._link_in(submodule) - return module + self._module._link_in(submodule) + return self._module + + @property + def module(self): + if self._module: + return self._module + return self.codegen(indent=self.indent) class File(CommonNode): """Holds a list of statements.""" _fields = ['body'] + _empty = None + - def __init__(self, name="generated", body=None): + def __init__(self, name="generated", body=None, path = None): self.name = name - self.body = body if body else [] + self.body = body or [] + self.config_target = 'c' + self.path = path or ctree.CONFIG.get('jit','COMPILE_PATH') + self._program_hash = None + + @property + def empty(self): + if not self._empty: + self._empty = type(self)(name=self.name, path=self.path).codegen() + return self._empty + + + @property + def path(self): + return self._path + + @path.setter + def path(self, value): + self._path = value + if not os.path.exists(self._path): + os.makedirs(self._path) + + def get_hash_filename(self): + return "%s.%s.sha" % (self.name, self._ext) + + + @property + def program_hash(self): + if not os.path.exists(os.path.join(self.path, self.get_hash_filename())): + return False + if self._program_hash: + return self._program_hash + with open(os.path.join(self.path, self.get_hash_filename())) as h_file: + return h_file.read().strip() + + @program_hash.setter + def program_hash(self, value): + self._program_hash = value + with open(os.path.join(self.path, self.get_hash_filename()), 'w') as h_file: + h_file.write(value) def codegen(self, *args): """Convert this substree into program text (a string).""" raise Exception("%s should override codegen()." % type(self)) - def _compile(self, program_text, compilation_dir): + def _compile(self, program_text, compilation_sub_dir): """Construct an LLVM module with the translated contents of this file.""" raise Exception("%s should override _compile()." % type(self)) @@ -210,6 +230,7 @@ def get_filename(self): class GeneratedPathRef(CommonNode): """Represents a path to a generated file.""" + _force_parentheses = False def __init__(self, target_file=None): assert isinstance(target_file, File), \ @@ -224,11 +245,15 @@ def visit_File(self, node): return ";\n".join(map(str, node.body)) + ";\n" def visit_GeneratedPathRef(self, node): - raise Exception("Unresolved GeneratedPathRefs to file %s." % (node.target.get_filename())) + return '"%s"'% (os.path.join(node.target.path, node.target.get_filename())) + # raise Exception("Unresolved GeneratedPathRefs to file %s." % (node.target.get_filename())) + + def visit_Pass(self, node): + return "" -class CommonDotGen(DotGenVisitor): +class CommonDotGen(DotGenLabeller): """Manages coversion of all common nodes to dot.""" - def label_GeneratedPathRef(self, node): + def visit_GeneratedPathRef(self, node): return "target: %s" % node.target.get_filename() diff --git a/ctree/np/__init__.py b/ctree/np/__init__.py new file mode 100644 index 0000000..e510994 --- /dev/null +++ b/ctree/np/__init__.py @@ -0,0 +1,75 @@ +import numpy as np +import ctypes as ct + +from ctree.types import ( + codegen_type, + register_type_recognizers, + register_type_codegenerators, +) + +def codegen_ndptr(ndptr): + prefix = "" + if getattr(ndptr, "_global", False): + prefix += "__global " + return prefix + "%s*" % codegen_type(ndptr._dtype_.type()) + +register_type_recognizers({ + np.ndarray: lambda obj: np.ctypeslib.as_ctypes(obj), + np.bool8: ct.c_bool, + + # signed integer types + np.byte: ct.c_char, + np.short: ct.c_short, + np.intc: ct.c_int, + np.longlong: ct.c_longlong, + + # technically not universally compatible + np.int8: ct.c_char, + np.int16: ct.c_short, + np.int32: ct.c_int, + np.int64: ct.c_long, + + # unsigned integer types + np.ubyte: ct.c_ubyte, + np.ushort: ct.c_ushort, + np.uintc: ct.c_uint, + np.ulonglong: ct.c_ulonglong, + + # floating point types + np.single: ct.c_float, + np.float32: ct.c_float, + np.double: ct.c_double, + np.float64: ct.c_double, +}) + +register_type_codegenerators({ + # pointers + np.ctypeslib._ndptr: codegen_ndptr, + + # boolean types + np.bool8: lambda t: "bool", + + # signed integer types + np.byte: lambda t: "char", + np.short: lambda t: "short", + np.intc: lambda t: "int", + np.longlong: lambda t: "long long", + + # technically not universally compatible + np.int8: lambda t: "char", + np.int16: lambda t: "short", + np.int32: lambda t: "int", + np.int64: lambda t: "long", + + # unsigned integer types + np.ubyte: lambda t: "unsigned byte", + np.ushort: lambda t: "unsigned short", + np.uintc: lambda t: "unsigned int", + np.ulonglong: lambda t: "unsigned long long", + + # floating point types + np.single: lambda t: "float", + np.float32: lambda t: "float", + np.double: lambda t: "double", + np.float64: lambda t: "double", +}) diff --git a/ctree/ocl/__init__.py b/ctree/ocl/__init__.py index 0a5d7ec..5e952c0 100644 --- a/ctree/ocl/__init__.py +++ b/ctree/ocl/__init__.py @@ -5,20 +5,39 @@ import logging log = logging.getLogger(__name__) +import ctree -# --------------------------------------------------------------------------- -# load OpenCL runtime into memory so it can be used from LLVM's jit -try: - import ctypes - import ctypes.util +if ctree.OCL_ENABLED: - libOpenCL = ctypes.util.find_library("OpenCL") - log.info("loading libOpenCL from %s", libOpenCL) + import pycl - import llvm.core + from ctree.types import ( + codegen_type, + register_type_recognizers, + register_type_codegenerators, + ) - llvm.core.load_library_permanently(libOpenCL) + register_type_recognizers({ + }) -except: - log.warn("Failed to load OpenCL runtime.") + register_type_codegenerators({ + pycl.cl_context: lambda t: "cl_context", + pycl.cl_command_queue: lambda t: "cl_command_queue", + pycl.cl_kernel: lambda t: "cl_kernel", + pycl.cl_mem: lambda t: "cl_mem", + }) + + + devices_context_queue_map = {} + + +def get_context_and_queue_from_devices(devices): + key = tuple(device.vendor_id for device in devices) + try: + return devices_context_queue_map[key] + except KeyError: + context = pycl.clCreateContext(devices) + queue = pycl.clCreateCommandQueue(context) + devices_context_queue_map[key] = (context, queue) + return devices_context_queue_map[key] diff --git a/ctree/ocl/codegen.py b/ctree/ocl/codegen.py index 543abaa..484563f 100644 --- a/ctree/ocl/codegen.py +++ b/ctree/ocl/codegen.py @@ -13,21 +13,3 @@ class OclCodeGen(CodeGenVisitor): def visit_OclFile(self, node): stmts = self._genblock(node.body, insert_curly_brackets=False, increase_indent=False) return '// %s' % (node.get_filename(), stmts) - - def visit_cl_device_id(self, node): - return "cl_device_id" - - def visit_cl_context(self, node): - return "cl_context" - - def visit_cl_command_queue(self, node): - return "cl_command_queue" - - def visit_cl_program(self, node): - return "cl_program" - - def visit_cl_kernel(self, node): - return "cl_kernel" - - def visit_cl_mem(self, node): - return "cl_mem" diff --git a/ctree/ocl/dotgen.py b/ctree/ocl/dotgen.py index 6bfbeda..a7c4a02 100644 --- a/ctree/ocl/dotgen.py +++ b/ctree/ocl/dotgen.py @@ -2,12 +2,12 @@ DOT generation for OpenCL. """ -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenLabeller -class OclDotGen(DotGenVisitor): +class OclDotLabeller(DotGenLabeller): """ Visitor to generator DOT. """ - def label_OclFile(self, node): + def visit_OclFile(self, node): return node.get_filename() diff --git a/ctree/ocl/macros.py b/ctree/ocl/macros.py index 821b590..4540501 100644 --- a/ctree/ocl/macros.py +++ b/ctree/ocl/macros.py @@ -3,11 +3,12 @@ programs. """ +import ast +from ctypes import c_size_t + from ctree.c.nodes import SymbolRef, Block, Assign, FunctionCall from ctree.c.nodes import If, Eq, NotEq, Or, Not, Ref, Constant, String -from ctree.c.types import Ptr, Long, FILE from ctree.c.macros import NULL, printf -from ctree.c.types import Int def CL_DEVICE_TYPE_GPU(): @@ -25,29 +26,114 @@ def CL_DEVICE_TYPE_ACCELERATOR(): def CL_DEVICE_TYPE_DEFAULT(): return SymbolRef("CL_DEVICE_TYPE_DEFAULT") + def CL_DEVICE_TYPE_ALL(): return SymbolRef("CL_DEVICE_TYPE_ALL") + def CL_SUCCESS(): return SymbolRef("CL_SUCCESS") + def CLK_LOCAL_MEM_FENCE(): return SymbolRef("CLK_LOCAL_MEM_FENCE") +def CLK_GLOBAL_MEM_FENCE(): + return SymbolRef("CLK_GLOBAL_MEM_FENCE") + + def barrier(arg): return FunctionCall(SymbolRef('barrier'), [arg]) + def get_local_id(id): return FunctionCall(SymbolRef('get_local_id'), [Constant(id)]) + def get_global_id(id): return FunctionCall(SymbolRef('get_global_id'), [Constant(id)]) + +def get_group_id(id): + return FunctionCall(SymbolRef('get_group_id'), [Constant(id)]) + + def get_local_size(id): return FunctionCall(SymbolRef('get_local_size'), [Constant(id)]) + +def get_global_size(id): + return FunctionCall(SymbolRef('get_global_size'), [Constant(id)]) + + def get_num_groups(id): return FunctionCall(SymbolRef('get_num_groups'), [Constant(id)]) + def clReleaseMemObject(arg): return FunctionCall(SymbolRef('clReleaseMemObject'), [arg]) + + +def clEnqueueWriteBuffer(queue, buf, blocking, offset, cb, ptr, num_events=0, evt_list_ptr=None, evt=None): + if isinstance(buf, str): buf = SymbolRef(buf) + if isinstance(blocking, bool): blocking = Constant(int(blocking)) + if isinstance(ptr, str): ptr = SymbolRef(ptr) + if not isinstance(offset, ast.AST): offset = Constant(offset) + if not isinstance(cb, ast.AST): cb = Constant(cb) + if not isinstance(num_events, ast.AST): num_events = Constant(num_events) + if not isinstance(evt_list_ptr, ast.AST): event_list_ptr = NULL() + if not isinstance(evt, ast.AST): evt = NULL() + return FunctionCall(SymbolRef('clEnqueueWriteBuffer'), [ + queue, buf, blocking, offset, cb, ptr, num_events, event_list_ptr, evt]) + +def clEnqueueReadBuffer(queue, buf, blocking, offset, cb, ptr, num_events=0, evt_list_ptr=None, evt=None): + if isinstance(buf, str): buf = SymbolRef(buf) + if isinstance(blocking, bool): blocking = Constant(int(blocking)) + if isinstance(ptr, str): ptr = SymbolRef(ptr) + if not isinstance(offset, ast.AST): offset = Constant(offset) + if not isinstance(cb, ast.AST): cb = Constant(cb) + if not isinstance(num_events, ast.AST): num_events = Constant(num_events) + if not isinstance(evt_list_ptr, ast.AST): event_list_ptr = NULL() + if not isinstance(evt, ast.AST): evt = NULL() + return FunctionCall(SymbolRef('clEnqueueReadBuffer'), [ + queue, buf, blocking, offset, cb, ptr, num_events, event_list_ptr, evt]) + +def clEnqueueCopyBuffer(queue, src_buf, dst_buf, src_offset=0, dst_offset=0, cb=0): + if isinstance(src_buf, str): src_buf = SymbolRef(src_buf) + if isinstance(dst_buf, str): dst_buf = SymbolRef(dst_buf) + if isinstance(src_offset, int): src_offset = Constant(src_offset) + if isinstance(dst_offset, int): dst_offset = Constant(dst_offset) + if isinstance(cb, int): cb = Constant(cb) + + num_events = Constant(0) + event_list_ptr = NULL() + evt = NULL() + + return FunctionCall(SymbolRef('clEnqueueCopyBuffer'), [ + queue, src_buf, dst_buf, src_offset, dst_offset, cb, num_events, event_list_ptr, evt]) + +def clSetKernelArg(kernel, arg_index, arg_size, arg_value): + if isinstance(kernel, str): kernel = SymbolRef(kernel) + if isinstance(arg_index, int): arg_index = Constant(arg_index) + if isinstance(arg_size, int): arg_size = Constant(arg_size) + if isinstance(arg_value, str): arg_value = Ref(SymbolRef(arg_value)) + return FunctionCall(SymbolRef("clSetKernelArg"), + [kernel, arg_index, arg_size, arg_value]) + +def clEnqueueNDRangeKernel(queue, kernel, work_dim=1, work_offset=0, global_size=0, local_size=0): + assert isinstance(queue, SymbolRef) + assert isinstance(kernel, SymbolRef) + global_size_sym = SymbolRef('global_size', c_size_t()) + local_size_sym = SymbolRef('local_size', c_size_t()) + call = FunctionCall(SymbolRef("clEnqueueNDRangeKernel"), [ + queue, kernel, + work_dim, work_offset, + Ref(global_size_sym.copy()), Ref(local_size_sym.copy()), + 0, NULL(), NULL() + ]) + + return Block([ + Assign(global_size_sym, Constant(global_size)), + Assign(local_size_sym, Constant(local_size)), + call + ]) diff --git a/ctree/ocl/nodes.py b/ctree/ocl/nodes.py index d2b6c66..3384079 100644 --- a/ctree/ocl/nodes.py +++ b/ctree/ocl/nodes.py @@ -3,6 +3,7 @@ """ from ctree.nodes import * +import hashlib class OclNode(CtreeNode): @@ -14,36 +15,44 @@ def codegen(self, indent=0): return OclCodeGen(indent).visit(self) - def _to_dot(self, indent=0): + def label(self, indent=0): """generate dot element for this node""" - from ctree.ocl.dotgen import OclDotGen + from ctree.ocl.dotgen import OclDotLabeller - return OclDotGen().visit(self) + return OclDotLabeller().visit(self) class OclFile(OclNode, File): """Represents a .cl file.""" + _ext = "cl" - def __init__(self, name="generated", body=None): - if not body: - body = [] + def __init__(self, name="generated", body=None, path = None): #TODO: Inspect complains about 2 args to __init__ - super(OclFile, self).__init__(name, body) - self._ext = "cl" + super(OclFile, self).__init__(name, body, path) - def _compile(self, program_text, compilation_dir): + def _compile(self, program_text): """ write the ocl program to a text file and compile it """ import os - cl_src_file = os.path.join(compilation_dir, self.get_filename()) - log.info("file for generated OpenCL: %s" % cl_src_file) - log.info("generated OpenCL code: (((\n%s\n)))" % program_text) + new_hash = hashlib.sha512(program_text.strip().encode()).hexdigest() + recreate_source = program_text != self._empty and new_hash != self.program_hash + self.program_hash = new_hash + cl_src_file = os.path.join(self.path, self.get_filename()) + if recreate_source: + log.info('Recreating source') + log.info("file for generated OpenCL: %s" % cl_src_file) + log.info("generated OpenCL code: (((\n%s\n)))" % program_text) + + # write program text to CL file + with open(cl_src_file, 'w') as cl_file: + cl_file.write(program_text) + else: + log.info("OpenCL file already generated") + return None - # write program text to C file - with open(cl_src_file, 'w') as cl_file: - cl_file.write(program_text) - - import llvm.core - - return llvm.core.Module.new("empty cl module") + def codegen(self, indent=0): + if self.body: + return super(OclFile, self).codegen(indent) + with open(os.path.join(self.path, self.get_filename())) as cl_file: + return cl_file.read() diff --git a/ctree/ocl/types.py b/ctree/ocl/types.py index 0e671d0..e69de29 100644 --- a/ctree/ocl/types.py +++ b/ctree/ocl/types.py @@ -1,39 +0,0 @@ -import ctypes - -from ctree.types import CtreeType, CtreeTypeResolver, TypeFetcher, get_ctree_type - - -class OclType(CtreeType): - """Base class for all built-in OpenCL Types.""" - - def codegen(self, indent=0): - from ctree.ocl.codegen import OclCodeGen - - return OclCodeGen().visit(self) - - def as_ctype(self): - raise NotImplementedError() - - -class cl_device_id(OclType): - pass - - -class cl_context(OclType): - pass - - -class cl_command_queue(OclType): - pass - - -class cl_program(OclType): - pass - - -class cl_kernel(OclType): - pass - - -class cl_mem(OclType): - pass diff --git a/ctree/omp/__init__.py b/ctree/omp/__init__.py index 452c3c1..1017db9 100644 --- a/ctree/omp/__init__.py +++ b/ctree/omp/__init__.py @@ -5,22 +5,3 @@ import logging log = logging.getLogger(__name__) - -# --------------------------------------------------------------------------- -# load omp runtime into memory so it can be used from LLVM's jit - -try: - import ctypes - import ctypes.util - - libiomp5 = ctypes.util.find_library("iomp5") - log.info("loading libiomp5 from %s" % libiomp5) - - import llvm.core - - llvm.core.load_library_permanently(libiomp5) - - #iomp_handle = ctypes.cdll.LoadLibrary(libiomp5) - #iomp_handle.__kmpc_barrier -except: - log.warn("Failed to load OpenMP runtime.") diff --git a/ctree/omp/codegen.py b/ctree/omp/codegen.py index 22f641a..6318080 100644 --- a/ctree/omp/codegen.py +++ b/ctree/omp/codegen.py @@ -22,6 +22,20 @@ def visit_OmpParallelFor(self, node): s += " " + ", ".join(map(str, node.clauses)) return s + def visit_OmpParallelSections(self, node): + s = "#pragma omp parallel sections" + if node.clauses: + s += " " + ", ".join(map(str, node.clauses)) + s += "\n%s%s" % (self._tab(), self._genblock(node.sections)) + return s + + def visit_OmpSection(self, node): + s = "#pragma omp section" + if node.clauses: + s += " " + ", ".join(map(str, node.clauses)) + s += "\n%s%s" % (self._tab(), self._genblock(node.body)) + return s + def visit_OmpIfClause(self, node): return "if(%s)" % node.exp diff --git a/ctree/omp/dotgen.py b/ctree/omp/dotgen.py index 5f83c1b..849cd5b 100644 --- a/ctree/omp/dotgen.py +++ b/ctree/omp/dotgen.py @@ -1,14 +1,13 @@ """ -DOT generation for OpenMP. +DOT labeller for OpenMP. """ -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenLabeller # --------------------------------------------------------------------------- # DOT generator - -class OmpDotGen(DotGenVisitor): +class OmpDotLabeller(DotGenLabeller): """ Visitor to generator DOT. """ diff --git a/ctree/omp/macros.py b/ctree/omp/macros.py index 45ef181..e55b442 100644 --- a/ctree/omp/macros.py +++ b/ctree/omp/macros.py @@ -2,8 +2,9 @@ Macros for using OpenMP. """ -from ctree.c.nodes import FunctionCall, SymbolRef +from ctree.c.nodes import FunctionCall, SymbolRef, Block from ctree.cpp.nodes import CppInclude +from ctree.omp.nodes import OmpParallelSections, OmpSection def omp_get_num_threads(): return FunctionCall(SymbolRef("omp_get_num_threads"), []) @@ -16,3 +17,24 @@ def omp_get_wtime(): def IncludeOmpHeader(): return CppInclude("omp.h") + + +def parallelize_tasks(dag): + """ + Returns an AST that computes the entries in dag in parallel using + omp sections. Dag must consist of: + 1) tuples, implying elements must be executed sequentially, + 2) frozensets, implying elements can be executed in parallel, + 3) ASTs, the contents themselves. + """ + if isinstance(dag, tuple): + sched = [] + for node in dag: + sched.extend(parallelize_tasks(node)) + return sched + elif isinstance(dag, frozenset): + sched = [OmpSection(body=parallelize_tasks(node)) for node in dag] + return [OmpParallelSections(sections=sched)] + else: + return [dag] + diff --git a/ctree/omp/nodes.py b/ctree/omp/nodes.py index b5fad1c..ee27304 100644 --- a/ctree/omp/nodes.py +++ b/ctree/omp/nodes.py @@ -20,10 +20,10 @@ def codegen(self, indent=0): return OmpCodeGen(indent).visit(self) - def _to_dot(self): - from ctree.omp.dotgen import OmpDotGen + def label(self): + from ctree.omp.dotgen import OmpDotLabeller - return OmpDotGen().visit(self) + return OmpDotLabeller().visit(self) def _requires_semicolon(self): return False @@ -47,6 +47,24 @@ def __init__(self, clauses=None): self.clauses = clauses if clauses else [] +class OmpParallelSections(OmpNode): + """ #pragma omp parallel sections... """ + _fields = ['clauses', 'sections'] + + def __init__(self, clauses=None, sections=None): + self.clauses = clauses or [] + self.sections = sections or [] + + +class OmpSection(OmpNode): + """ #pragma omp section ... """ + _fields = ['clauses', 'body'] + + def __init__(self, clauses=None, body=None): + self.clauses = clauses or [] + self.body = body or [] + + class OmpIvDep(OmpNode): _field = ['clauses'] diff --git a/ctree/opentuner/driver.py b/ctree/opentuner/driver.py index 7083f19..b2a9d60 100644 --- a/ctree/opentuner/driver.py +++ b/ctree/opentuner/driver.py @@ -14,6 +14,7 @@ from opentuner.tuningrunmain import TuningRunMain from opentuner.search.manipulator import ConfigurationManipulator from opentuner.measurement.inputmanager import FixedInputManager +from opentuner.api import TuningRunManager class OpenTunerDriver(TuningDriver): @@ -28,60 +29,38 @@ def __init__(self, *ot_args, **ot_kwargs): to run the tuning logic. """ super(OpenTunerDriver, self).__init__() - self._results = queue.Queue(1) - self._configs = queue.Queue(1) self._best_config = None - self._thread = OpenTunerThread(self, *ot_args, **ot_kwargs) - self._thread.start() + interface = CtreeMeasurementInterface(self, *ot_args, **ot_kwargs) + arg_parser = argparse.ArgumentParser(parents=opentuner.argparsers()) + config_args = CONFIG.get("opentuner", "args").split() + tuner_args = arg_parser.parse_args(config_args) + self.manager = TuningRunManager(interface, tuner_args) self._converged = False def _get_configs(self): """Get the next configuration to test.""" timeout = CONFIG.getint("opentuner", "timeout") while True: - try: - yield self._configs.get(True, timeout) - except queue.Empty: + self.curr_desired_result = self.manager.get_next_desired_result() + if self.curr_desired_result is None: break + yield self.curr_desired_result.configuration.data + print("Best configuration", self.manager.get_best_configuration()) log.info("exhausted stream of configurations.") - assert self._best_config != None, "No best configuration reported." + best_config = self.manager.get_best_configuration() + assert best_config != None, "No best configuration reported." self._converged = True while True: - yield self._best_config + yield best_config def report(self, **kwargs): """Report the performance of the most recent configuration.""" if not self._converged: - result = Result(**kwargs) - self._results.put_nowait(result) - - -class OpenTunerThread(threading.Thread): - """ - Thread to drive OpenTuner. - """ - def __init__(self, driver, *ot_args, **ot_kwargs): - super(OpenTunerThread, self).__init__() - self._ctree_driver = driver - self._ot_args = ot_args - self._ot_kwargs = ot_kwargs - self._tuningrun = None - - # variables for Thread class - self.name = "opentuner_driver" - self.daemon = True - - def run(self): - """Starts the main OpenTuner loop.""" - log.info("tuning thread '%s' starting (%d total threads now).", \ - self.name, threading.active_count()) - arg_parser = argparse.ArgumentParser(parents=opentuner.argparsers()) - config_args = CONFIG.get("opentuner", "args").split() - tuner_args = arg_parser.parse_args(config_args) - interface = CtreeMeasurementInterface(self._ctree_driver, *self._ot_args, **self._ot_kwargs) - TuningRunMain(interface, tuner_args).main() - log.info("tuning thread '%s' terminating.", self.name) + print("Tuning run result:", self.curr_desired_result.configuration.data, kwargs) + self.manager.report_result(self.curr_desired_result, Result(**kwargs)) + # result = Result(**kwargs) + # self._results.put_nowait(result) class CtreeMeasurementInterface(MeasurementInterface): diff --git a/ctree/py/dotgen.py b/ctree/py/dotgen.py index f8b8c70..8019653 100644 --- a/ctree/py/dotgen.py +++ b/ctree/py/dotgen.py @@ -7,25 +7,31 @@ # --------------------------------------------------------------------------- # dot generator -from ctree.dotgen import DotGenVisitor, to_dot +from ctree.dotgen import DotGenLabeller -class PyDotGen(DotGenVisitor): +class PyDotLabeller(DotGenLabeller): # pragma: no cover """ Manages generation of DOT. """ - def label_arg(self, node): + def visit_arg(self, node): s = "name: %s" % node.arg if node.annotation and not isinstance(node.annotation, ast.AST): s += "\nannotation: %s" % self._qualified_name(node.annotation) return s - def label_FunctionDef(self, node): + def visit_FunctionDef(self, node): return "name: %s" % node.name - def label_Num(self, node): + def visit_Num(self, node): return "n: %s" % node.n - def label_Name(self, node): + def visit_Name(self, node): return "id: %s" % node.id + + def visit_Attribute(self, node): + return "attr: %s" % node.attr + + def visit_Str(self, node): + return "str: %s" % node.s diff --git a/ctree/simd/__init__.py b/ctree/simd/__init__.py index e69de29..751344e 100644 --- a/ctree/simd/__init__.py +++ b/ctree/simd/__init__.py @@ -0,0 +1,9 @@ +from ctree.simd.types import m256d, m256, m512 + +from ctree.types import register_type_codegenerators + +register_type_codegenerators({ + m256d: lambda t: "__m256d", + m256: lambda t: "__m256", + m512: lambda t: "__m512" +}) diff --git a/ctree/simd/codegen.py b/ctree/simd/codegen.py index 9d8e301..a4da6e1 100644 --- a/ctree/simd/codegen.py +++ b/ctree/simd/codegen.py @@ -11,3 +11,6 @@ class SimdCodeGen(CodeGenVisitor): """ def visit_m256d(self, node): return "__m256d" + + def visit_m256(self, node): + return "__m256" diff --git a/ctree/simd/dotgen.py b/ctree/simd/dotgen.py index 23fab87..ffb2fd0 100644 --- a/ctree/simd/dotgen.py +++ b/ctree/simd/dotgen.py @@ -1,12 +1,12 @@ """ -DOT generation for SIMD. +DOT labeller for SIMD. """ -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenLabeller -class SimdDotGen(DotGenVisitor): +class SimdDotLabeller(DotGenLabeller): """ - Visitor to generator DOT. + Visitor to label SIMD nodes in DOT. """ pass diff --git a/ctree/simd/macros.py b/ctree/simd/macros.py index 4c32aaa..707f641 100644 --- a/ctree/simd/macros.py +++ b/ctree/simd/macros.py @@ -8,7 +8,7 @@ def _make_call(name, nArgs): def _the_call(*args): assert len(args) == nArgs, \ "Macro expected %d args, got %d." % (nArgs, len(args)) - return FunctionCall(SymbolRef(name), args) + return FunctionCall(SymbolRef(name), list(args)) return _the_call mm256_storeu_pd = _make_call("_mm256_storeu_pd", 2) @@ -17,3 +17,11 @@ def _the_call(*args): mm256_set1_pd = _make_call("_mm256_set1_pd", 1) mm256_add_pd = _make_call("_mm256_add_pd", 2) mm256_mul_pd = _make_call("_mm256_mul_pd", 2) + +mm256_load_ps = _make_call("_mm256_load_ps", 1) +mm256_store_ps = _make_call("_mm256_store_ps", 2) +mm256_set1_ps = _make_call("_mm256_set1_ps", 1) + +mm512_load_ps = _make_call("_mm512_load_ps", 1) +mm512_store_ps = _make_call("_mm512_store_ps", 2) +mm512_set1_ps = _make_call("_mm512_set1_ps", 1) diff --git a/ctree/simd/nodes.py b/ctree/simd/nodes.py index a0680d0..5605fd7 100644 --- a/ctree/simd/nodes.py +++ b/ctree/simd/nodes.py @@ -13,7 +13,7 @@ def codegen(self, indent=0): return SimdCodeGen(indent).visit(self) - def _to_dot(self, _): - from ctree.sse.dotgen import SimdDotGen + def label(self): + from ctree.sse.dotgen import SimdDotLabeller - return SimdDotGen().visit(self) + return SimdDotLabeller().visit(self) diff --git a/ctree/simd/types.py b/ctree/simd/types.py index 5981ccb..88f36c9 100644 --- a/ctree/simd/types.py +++ b/ctree/simd/types.py @@ -1,7 +1,4 @@ -from ctree.types import CtreeType - - -class SimdType(CtreeType): +class SimdType(object): """Base class for all SIMD Types.""" def codegen(self, indent=0): @@ -9,9 +6,12 @@ def codegen(self, indent=0): return SimdCodeGen().visit(self) - def as_ctype(self): - raise NotImplementedError() - class m256d(SimdType): pass + +class m256(SimdType): + pass + +class m512(SimdType): + pass diff --git a/ctree/templates/dotgen.py b/ctree/templates/dotgen.py index b0bc4ea..0252eb4 100644 --- a/ctree/templates/dotgen.py +++ b/ctree/templates/dotgen.py @@ -4,16 +4,16 @@ import os -from ctree.dotgen import DotGenVisitor +from ctree.dotgen import DotGenLabeller -class TemplateDotGen(DotGenVisitor): +class TemplateDotLabeller(DotGenLabeller): """ Visitor to generator DOT. """ - def label_StringTemplate(self, node): + def visit_StringTemplate(self, node): return "template: <<<\n%s\n>>>" % \ node._template.template.replace("\n", "\\n").replace('"', r"\"") - def label_FileTemplate(self, node): + def visit_FileTemplate(self, node): return os.path.basename(node._template_path) diff --git a/ctree/templates/nodes.py b/ctree/templates/nodes.py index 1d99002..c9ea410 100644 --- a/ctree/templates/nodes.py +++ b/ctree/templates/nodes.py @@ -26,10 +26,10 @@ def codegen(self, indent=0): return TemplateCodeGen(indent).visit(self) - def _to_dot(self): - from ctree.templates.dotgen import TemplateDotGen + def label(self): + from ctree.templates.dotgen import TemplateDotLabeller - return TemplateDotGen().visit(self) + return TemplateDotLabeller().visit(self) def _requires_semicolon(self): return False diff --git a/ctree/tools/generators/builder.py b/ctree/tools/generators/builder.py index 0fc6ebd..7ac07ea 100644 --- a/ctree/tools/generators/builder.py +++ b/ctree/tools/generators/builder.py @@ -57,8 +57,8 @@ def indent_print(s): try: os.makedirs(target_dir) except OSError as exception: - print "Unable to create %s error (%d) %s" % \ - (target_dir,exception.errno,exception.strerror) + print ("Unable to create %s error (%d) %s" % \ + (target_dir,exception.errno,exception.strerror)) exit(1) if target_dir[-4:] == '/bin': diff --git a/ctree/tools/generators/templates/create/tests/.gitignore b/ctree/tools/generators/templates/create/tests/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/ctree/tools/runner.py b/ctree/tools/runner.py index 0d39c12..e4e539d 100644 --- a/ctree/tools/runner.py +++ b/ctree/tools/runner.py @@ -3,9 +3,23 @@ basically copies all files and directories from a template. """ +from __future__ import print_function import sys import argparse -from ctree.tools.generators import builder as Builder +import collections +import shutil +import os + +import ctree +from ctree.tools.generators.builder import Builder + + +if sys.version_info >= (3, 0, 0): # python 3 + # noinspection PyPep8Naming + import configparser as ConfigParser +else: + # noinspection PyPep8Naming + import ConfigParser __author__ = 'chick' @@ -24,24 +38,129 @@ def main(*args): ) parser.add_argument('-p', '--port', help="/dev name to use for wattsup meter port") parser.add_argument('-v', '--verbose', help='show more debug than you like', action="store_true") + parser.add_argument('-dc', '--disable_cache', help='disable and delete the persistent cache', action="store_true") + parser.add_argument('-ec', '--enable_cache', help='enable the persistent cache', action="store_true") + parser.add_argument('-cc', '--clear_cache', help='clear the persistent cache', action="store_true") args = parser.parse_args(args) if args.startproject: specializer_name = args.startproject - print "create project specializer %s" % specializer_name - - builder = Builder.Builder("create", specializer_name, verbose=args.verbose) + print("create project specializer %s" % specializer_name) + builder = Builder("create", specializer_name, verbose=args.verbose) builder.build(None, None) + elif args.wattsupmeter: from ctree.metrics.watts_up_reader import WattsUpReader port = args.port if args.port else WattsUpReader.guess_port() meter = WattsUpReader(port_name=port) meter.interactive_mode() + + elif args.enable_cache: + ctree.CONFIG.set("jit", "CACHE", value="True") + write_success = write_to_config('jit', 'CACHE', True) + if write_success: + print("[SUCCESS] ctree caching enabled.") + + elif args.disable_cache: + wipe_cache() + ctree.CONFIG.set("jit", "CACHE", value="False") + write_success = write_to_config('jit', 'CACHE', False) + args.clear_cache = True + if write_success: + print("[SUCCESS] ctree caching disabled.") + + elif args.clear_cache: + wipe_cache() + else: parser.print_usage() + +def get_responsible(section, key): + """ + :param section: Section to search for + :param key: key to search for + :return: path of config file responsible for setting + """ + first = ctree.CFG_PATHS[-1] + paths = reversed(ctree.CFG_PATHS) + for path in paths: + config = ConfigParser.ConfigParser() + config.read(path) + if config.has_option(section, key): + return path + return first + + +def write_to_config(section, key, value): + """ + This method handles writing to the closest config file to the current + project, but does not write to the defaults.cfg file in ctree. + :return: return True if write is successful. False otherwise. + """ + + if ctree.CFG_PATHS: + target = get_responsible(section, key) + config = ConfigParser.ConfigParser() + config.read(target) + print(target) + if not config.has_section(section): + config.add_section(section) + config.set(section, key, value) + with open(target, 'w') as configfile: + config.write(configfile) + configfile.close() + return True + else: + print("[FAILURE] No config file detected. Please create a '.ctree.cfg' file in your project directory.") + return False + + +def wipe_cache(): + """ + if path is absolute, just remove the directory + if the path is relative, recursively look from current directory down + looking for matching paths. This can take a long time looking for + :return: + """ + cache_name = os.path.expanduser(ctree.CONFIG.get('jit', 'COMPILE_PATH')) + if os.path.isabs(cache_name): + if os.path.exists(cache_name): + result = shutil.rmtree(cache_name) + print("removed cache directory {} {}".format( + cache_name, result if result else "")) + exit(0) + + + splitted = cache_name.split(os.sep) + while splitted: + first = splitted[0] + if first == '.': + splitted.pop(0) + elif first == '..': + os.chdir('../') + splitted.pop(0) + else: + cache_name = os.sep.join(splitted) + break + + wipe_queue = collections.deque([os.path.abspath(p) for p in os.listdir(os.getcwd())]) + print("ctree looking for relative cache directories named {}, checking directories under this one".format( + cache_name)) + while wipe_queue: + directory = wipe_queue.popleft() + if not os.path.isdir(directory): + continue + if os.path.split(directory)[-1] == cache_name: + shutil.rmtree(directory) + else: + #print("{} ".format(directory)) + for sub_item in os.listdir(directory): + wipe_queue.append(os.path.join(directory, sub_item)) + print() + if __name__ == '__main__': main(sys.argv[1:]) diff --git a/ctree/transformations.py b/ctree/transformations.py index 2cf109c..fabe385 100644 --- a/ctree/transformations.py +++ b/ctree/transformations.py @@ -2,15 +2,40 @@ A set of basic transformers for python asts """ import os +import sys import ast +from ctypes import c_long, c_int, c_byte, c_short, c_char_p, c_void_p, c_float +import ctypes +from collections import deque -from ctree.nodes import Project, CtreeNode -from ctree.c.nodes import Op, Constant, String, SymbolRef, BinaryOp, TernaryOp, Return -from ctree.c.nodes import If, CFile, FunctionCall, FunctionDecl, For, Assign, AugAssign -from ctree.c.nodes import Lt, PostInc, AddAssign, SubAssign, MulAssign, DivAssign -from ctree.c.types import Long +import ctree +from ctree.c.nodes import Constant, String, SymbolRef, BinaryOp, TernaryOp, \ + Return, While, MultiNode, UnaryOp +from ctree.c.nodes import If, CFile, FunctionCall, FunctionDecl, For, Assign, \ + ArrayRef, Cast +from ctree.nodes import CtreeNode +from ctree.c.nodes import Lt, Gt, AddAssign +from ctree.c.nodes import Break, Continue, Pass, Array, Literal, And +from ctree.c.nodes import Op from ctree.visitors import NodeTransformer -from ctree.util import flatten + +from ctree.types import get_common_ctype + + +# conditional imports + +if sys.version_info < (3, 0): + from itertools import izip_longest +else: + from itertools import zip_longest as izip_longest + + +def get_type(node): + if hasattr(node, 'get_type'): + return type(node.get_type()) + elif hasattr(node, 'type'): + return type(node.type) + return c_void_p class PyCtxScrubber(NodeTransformer): @@ -28,15 +53,46 @@ class PyBasicConversions(NodeTransformer): """ Convert constructs with obvious C analogues. """ + def __init__(self, names_dict={}, constants_dict={}): + self.names_dict = names_dict + self.constants_dict = constants_dict + PY_OP_TO_CTREE_OP = { ast.Add: Op.Add, ast.Mod: Op.Mod, ast.Mult: Op.Mul, ast.Sub: Op.Sub, + ast.Div: Op.Div, ast.Lt: Op.Lt, + ast.Gt: Op.Gt, + ast.LtE: Op.LtE, + ast.GtE: Op.GtE, + ast.BitAnd: Op.BitAnd, + ast.BitOr: Op.BitOr, + ast.Eq: Op.Eq, + ast.NotEq: Op.NotEq, + ast.Not: Op.Not, + ast.And: Op.And, + ast.Or: Op.Or, + ast.BitXor: Op.BitXor, + ast.LShift: Op.BitShL, + ast.RShift: Op.BitShR, + ast.Is: Op.Eq, + ast.IsNot: Op.NotEq, + ast.USub: Op.SubUnary, + ast.UAdd: Op.AddUnary, + ast.FloorDiv: Op.Div, + ast.Invert: Op.BitNot # TODO list the rest } + PY_UOP_TO_CTREE_UOP = { + 'UAdd': Op.Add, + 'USub': Op.Sub, + 'Not': Op.Not, + 'Invert': Op.BitNot + } + def visit_Num(self, node): return Constant(node.n) @@ -44,12 +100,16 @@ def visit_Str(self, node): return String(node.s) def visit_Name(self, node): + if node.id in self.constants_dict: + return Constant(self.constants_dict[node.id]) + if node.id in self.names_dict: + return SymbolRef(self.names_dict[node.id]) return SymbolRef(node.id) def visit_BinOp(self, node): lhs = self.visit(node.left) rhs = self.visit(node.right) - op = self.PY_OP_TO_CTREE_OP[type(node.op)]() + op = self.PY_OP_TO_CTREE_OP.get(type(node.op), type(node.op))() return BinaryOp(lhs, op, rhs) def visit_Return(self, node): @@ -63,7 +123,7 @@ def visit_For(self, node): if isinstance(node, ast.For) and \ isinstance(node.iter, ast.Call) and \ isinstance(node.iter.func, ast.Name) and \ - node.iter.func.id == 'range': + node.iter.func.id in ('range', 'xrange'): Range = node.iter nArgs = len(Range.args) if nArgs == 1: @@ -75,17 +135,46 @@ def visit_For(self, node): elif nArgs == 3: start, stop, step = map(self.visit, Range.args) else: - raise Exception("Cannot convert a for...range with %d args." % nArgs) + raise Exception( + "Cannot convert a for...range with %d args." % nArgs) + + # check no-op conditions. + if all(isinstance(item, Constant) for item in (start, stop, step)): + if step.value == 0: + raise ValueError("range() step argument must not be zero") + elif start.value == stop.value or \ + (start.value < stop.value and step.value < 0) or \ + (start.value > stop.value and step.value > 0): + return None + + if not all(isinstance(item, CtreeNode) + for item in (start, stop, step)): + node.body = list(map(self.visit, node.body)) + return node # TODO allow any expressions castable to Long type - assert stop.get_type() == Long(), "Can only convert range's with stop values of Long type." - assert start.get_type() == Long(), "Can only convert range's with start values of Long type." - assert step.get_type() == Long(), "Can only convert range's with step values of Long type." + target_types = [c_long] + for el in (stop, start, step): + # typed item to try and guess type off of. Imperfect right now. + if hasattr(el, 'get_type'): + # TODO take the proper class instead of the last; if start, + # end are doubles, but step is long, target is double + t = el.get_type() + assert any(isinstance(t, klass) for klass in [ + c_byte, c_int, c_long, c_short + ]), "Can only convert ranges with integer/long \ + start/stop/step values" + target_types.append(type(t)) + target_type = get_common_ctype(target_types)() - target = SymbolRef(node.target.id, Long()) + target = SymbolRef(node.target.id, target_type) + op = Lt + if hasattr(start, 'value') and hasattr(stop, 'value') and \ + start.value > stop.value: + op = Gt for_loop = For( Assign(target, start), - Lt(target.copy(), stop), + op(target.copy(), stop), AddAssign(target.copy(), step), [self.visit(stmt) for stmt in node.body], ) @@ -108,23 +197,50 @@ def visit_IfExp(self, node): elze = self.visit(node.orelse) return TernaryOp(cond, then, elze) + def visit_BoolOp(self, node): + first = self.visit(node.values[0]) + second = self.visit(node.values[1]) + op = self.PY_OP_TO_CTREE_OP.get(type(node.op), + type(node.op))() + curr = BinaryOp(first, op, second) + for value in node.values[2:]: + curr = BinaryOp(curr, op, self.visit(value)) + return curr + def visit_Compare(self, node): - assert len(node.ops) == 1, \ - "PyBasicConversions doesn't support Compare nodes with more than one operator." lhs = self.visit(node.left) - op = self.PY_OP_TO_CTREE_OP[type(node.ops[0])]() + + op = self.PY_OP_TO_CTREE_OP.get(type(node.ops[0]), + type(node.ops[0]))() rhs = self.visit(node.comparators[0]) - return BinaryOp(lhs, op, rhs) + curr = BinaryOp(lhs, op, rhs) + for i in range(1, len(node.ops)): + op = self.PY_OP_TO_CTREE_OP.get(type(node.ops[i]), + type(node.ops[i]))() + rhs = self.visit(node.comparators[i]) + lhs = self.visit(node.comparators[i-1]) + curr = And(curr, BinaryOp(lhs, op, rhs)) + return curr def visit_Module(self, node): body = [self.visit(s) for s in node.body] - return Project([CFile("module", body)]) + return CFile("module", body) def visit_Call(self, node): args = [self.visit(a) for a in node.args] fn = self.visit(node.func) + if getattr(node, 'starargs', None) is not None: + node.func = fn + node.args = args + node.starargs = self.visit(node.starargs) + return node + if hasattr(fn, "name") and fn.name == "float": + return Cast(c_float(), args[0]) return FunctionCall(fn, args) + def visit_Expr(self, node): + return self.visit(node.value) + def visit_FunctionDef(self, node): if ast.get_docstring(node): node.body.pop(0) @@ -139,36 +255,154 @@ def visit_AugAssign(self, node): op = type(node.op) target = self.visit(node.target) value = self.visit(node.value) - if op is ast.Add: - return AddAssign(target, value) - elif op is ast.Sub: - return SubAssign(target, value) - elif op is ast.Mult: - return MulAssign(target, value) - elif op is ast.Div: - return DivAssign(target, value) - # Error? + # if op is ast.Add: + # return AddAssign(target, value) + # elif op is ast.Sub: + # return SubAssign(target, value) + # elif op is ast.Mult: + # return MulAssign(target, value) + # elif op is ast.Div: + # return DivAssign(target, value) + # elif op is ast.BitXor: + # return BitXorAssign(target, value) + # # TODO: Error? + lookup = { + ast.Add: 'AddAssign', ast.Sub: 'SubAssign', ast.Mult: 'MulAssign', + ast.Div: 'DivAssign', ast.BitAnd: 'BitAndAssign', ast.BitOr: + 'BitOrAssign', ast.BitXor: 'BitXorAssign', ast.LShift: + 'BitShLAssign', ast.RShift: 'BitShRAssign' + } + if op in lookup: + return getattr(ctree.c.nodes, lookup[op])(target, value) return node + def targets_to_list(self, targets): + """parses target into nested lists""" + res = [] + for elt in targets: + if not isinstance(elt, (ast.List, ast.Tuple)): + res.append(elt) + elif isinstance(elt, (ast.Tuple, ast.List)): + res.append(self.targets_to_list(elt.elts)) + return res -class FixUpParentPointers(NodeTransformer): - """ - Add parent pointers if they're missing. - """ + def value_to_list(self, value): + """parses value into nested lists for multiple assign""" + res = [] + if not isinstance(value, (ast.List, ast.Tuple)): + return value + for elt in value.elts: + if not isinstance(value, (ast.List, ast.Tuple)): + res.append(elt) + else: + res.append(self.value_to_list(elt)) + return ast.List(elts=res) - def generic_visit(self, node): - for fieldname, value in ast.iter_fields(node): - for child in flatten(value): - if isinstance(child, CtreeNode): - setattr(child, 'parent', node) - self.visit(child) - return node + def pair_lists(self, targets, values): + res = [] + queue = deque((target, values) for target in targets) + sentinel = object() + while queue: + target, value = queue.popleft() + if isinstance(target, list): + # target hasn't been completely unrolled yet + for sub_target, sub_value in izip_longest( + target, value.elts, fillvalue=sentinel): + if sub_target is sentinel or \ + sub_value is sentinel: + raise ValueError( + 'Incorrect number of values to unpack') + queue.append((sub_target, sub_value)) + else: + res.append((target, value)) + return res + + def parse_pairs(self, node): + targets = self.targets_to_list(node.targets) + values = self.value_to_list(node.value) + return self.pair_lists(targets, values) + + def visit_Assign(self, node): + + target_value_list = [(self.visit(target), self.visit(value)) + for target, value in self.parse_pairs(node)] + + if len(target_value_list) == 1: + target, value = target_value_list[0] + return Assign(target, value) + + operation_body = [] + swap_body = [] + for target, value in target_value_list: + if not isinstance(target, SymbolRef): + operation_body.append(Assign(target, value)) + elif isinstance(value, Literal) and \ + not isinstance(value, SymbolRef): + operation_body.append(Assign(target, value)) + else: + new_target = target.copy() + new_target.name = "____temp__" + new_target.name + operation_body.append(Assign(new_target, value)) + swap_body.append(Assign(target, new_target.copy())) + return MultiNode(body=operation_body + swap_body) + + def visit_Subscript(self, node): + if isinstance(node.slice, ast.Index): + value = self.visit(node.value) + index = self.visit(node.slice.value) + return ArrayRef(value, index) + else: + return node + + def visit_While(self, node): + cond = self.visit(node.test) + body = [self.visit(i) for i in node.body] + return While(cond, body) + + def visit_Lambda(self, node): + + if isinstance(node, ast.Lambda): + def_node = ast.FunctionDef(name="default", args=node.args, + body=node.body, decorator_list=None) + + params = [self.visit(p) for p in def_node.args.args] + defn = [Return(self.visit(def_node.body))] + decl_node = FunctionDecl(None, def_node.name, params, defn) + Lifter().visit_FunctionDecl(decl_node) + + return decl_node + else: + return node + + def visit_Break(self, node): + return Break() + + def visit_Continue(self, node): + return Continue() + + def visit_Pass(self, node): + return Pass() + + def visit_List(self, node): + elts = [self.visit(elt) for elt in node.elts] + types = [get_type(elt) for elt in elts] + array_type = get_common_ctype(types) + return Array(type=ctypes.POINTER(array_type)(), body=elts) + + def visit_UnaryOp(self, node): + # If it's already C unary op, recurse only + if isinstance(node, UnaryOp): + node.arg = self.visit(node.arg) + return node + argument = self.visit(node.operand) + op = self.PY_OP_TO_CTREE_OP.get(type(node.op), type(node.op))() + return UnaryOp(op, argument) class ResolveGeneratedPathRefs(NodeTransformer): """ - Converts any instances of ctree.nodes.GeneratedPathRef into strings containing the absolute path - of the target file. + Converts any instances of ctree.nodes.GeneratedPathRef into strings + containing the absolute path of the target file. """ def __init__(self, compilation_dir): @@ -177,4 +411,35 @@ def __init__(self, compilation_dir): def visit_GeneratedPathRef(self, node): self.count += 1 - return String(os.path.join(self.compilation_dir, node.target.get_filename())) + return String(os.path.join(self.compilation_dir, + node.target.get_filename())) + + +class Lifter(NodeTransformer): + """ + To aid in adding new includes or parameters during tree + traversals, users can store them with arbitrary child nodes and call this + transformation to move them to the correct position. + """ + def __init__(self, lift_params=True, lift_includes=True): + self.lift_params = lift_params + self.lift_includes = lift_includes + + def visit_FunctionDecl(self, node): + if self.lift_params: + for child in ast.walk(node): + for param in getattr(child, '_lift_params', []): + if param not in node.params: + node.params.append(param) + # del child._lift_params + return self.generic_visit(node) + + def visit_CFile(self, node): + if self.lift_includes: + new_includes = [] + for child in ast.walk(node): + for include in getattr(child, '_lift_includes', []): + if include not in new_includes: + new_includes.append(include) + node.body = list(new_includes) + node.body + return self.generic_visit(node) diff --git a/ctree/transforms/__init__.py b/ctree/transforms/__init__.py new file mode 100644 index 0000000..1a88c64 --- /dev/null +++ b/ctree/transforms/__init__.py @@ -0,0 +1,2 @@ +from ctree.transforms.constant_fold import ConstantFold +from ctree.transforms.declaration_filler import DeclarationFiller diff --git a/ctree/transforms/constant_fold.py b/ctree/transforms/constant_fold.py new file mode 100644 index 0000000..a76c634 --- /dev/null +++ b/ctree/transforms/constant_fold.py @@ -0,0 +1,54 @@ +import ctree.c.nodes as C +import ast + + +op_map = { + C.Op.Add: lambda x, y: x + y, + C.Op.Div: lambda x, y: x / y, + C.Op.Mul: lambda x, y: x * y, + C.Op.Lt: lambda x, y: x < y, + C.Op.Sub: lambda x, y: x - y, +} + + +class ConstantFold(ast.NodeTransformer): + """ TODO: Support all folding situations """ + def fold_add(self, node): + if isinstance(node.left, C.Constant) and node.left.value == 0: + return node.right + elif isinstance(node.right, C.Constant) and node.right.value == 0: + return node.left + return node + + def fold_sub(self, node): + if isinstance(node.left, C.Constant) and node.left.value == 0: + return C.Sub(node.right) + elif isinstance(node.right, C.Constant) and node.right.value == 0: + return node.left + return node + + def fold_mul(self, node): + if isinstance(node.left, C.Constant) and node.left.value == 1: + return node.right + elif isinstance(node.right, C.Constant) and node.right.value == 1: + return node.left + elif isinstance(node.left, C.Constant) and node.left.value == 0: + return node.left + elif isinstance(node.right, C.Constant) and node.right.value == 0: + return node.right + return node + + def visit_BinaryOp(self, node): + node.left = self.visit(node.left) + node.right = self.visit(node.right) + if isinstance(node.left, C.Constant) and \ + isinstance(node.right, C.Constant): + return C.Constant(op_map[node.op.__class__]( + node.left.value, node.right.value)) + elif isinstance(node.op, C.Op.Add): + return self.fold_add(node) + elif isinstance(node.op, C.Op.Sub): + return self.fold_sub(node) + elif isinstance(node.op, C.Op.Mul): + return self.fold_mul(node) + return node diff --git a/ctree/transforms/declaration_filler.py b/ctree/transforms/declaration_filler.py new file mode 100644 index 0000000..d9694c2 --- /dev/null +++ b/ctree/transforms/declaration_filler.py @@ -0,0 +1,131 @@ +import ast +import ctree.c.nodes as C +import ctypes as ct + + +class DeclarationFiller(ast.NodeTransformer): + default_function_retvals = { + 'fmin': ct.c_double(), + 'fmax': ct.c_double(), + 'fabs': ct.c_double() + } + + tmp_prefix = "____temp__" + + def __init__(self): + self.__environments = [self.default_function_retvals.copy()] + + def _lookup(self, key): + """ + :param key: + :return: Looks up the last value corresponding to key in + self.__environments + """ + if isinstance(key, C.SymbolRef): + key = key.name + value = sentinel = object() + for environment in self.__environments: + if key in environment: + value = environment[key] + if value is sentinel: + raise KeyError('Did not find {} in environments'.format(repr(key))) + return value + + def _has_key(self, key): + try: + self._lookup(key) + return True + except KeyError: + return False + + def __add_entry(self, key, value): + if isinstance(key, C.SymbolRef): + key = key.name + self.__environments[-1][key] = value + + def __add_environment(self): + self.__environments.append({}) + + def __pop_environment(self): + return self.__environments.pop() + + def visit_FunctionDecl(self, node): + # add current FunctionDecl's return type onto environments + self.__add_entry(node.name, node.return_type) + + # new environment every time we enter a function + self.__add_environment() + + for param in node.params: + # binding types of parameters + self.__add_entry(param.name, param.type) + + node.defn = [self.visit(i) for i in node.defn] + self.__pop_environment() + return node + + def visit_For(self, node): + self.__add_environment() + self.generic_visit(node) + self.__pop_environment() + return node + + def visit_SymbolRef(self, node): + + if node.type is not None: + self.__add_entry(node.name, node.type) + return node + + def visit_FunctionCall(self, node): + node.args = [self.visit(arg) for arg in node.args] + if self._has_key(node.func): + node.type = self._lookup(node.func) + elif node.func.name == 'float': + node.type = ct.c_float() + elif node.func.name in {'fmax', 'fmin'}: + # Assume type of last argument for now + # TODO: Is there something smarter we can do? + if isinstance(node.args[0], C.SymbolRef): + node.type = self._lookup(node.args[0]) + elif hasattr(node.args[0], 'get_type'): + node.type = node.args[0].get_type(self) + else: + raise NotImplementedError(node.args[0]) + return node + + def visit_BinaryOp(self, node): + if isinstance(node.op, C.Op.Assign): + node.left = self.visit(node.left) + if isinstance(node.left, C.BinaryOp): + return node + node.right = self.visit(node.right) + name = node.left + value = node.right + if hasattr(name, 'type') and name.type is not None: + return node + if hasattr(name, 'name') and not self._has_key(name.name): + # temporary variable types can be derived from the variables + # that they represent + if name.name.startswith('____temp__'): + stripped_name = name.name[len(self.tmp_prefix):] + #print(stripped_name) + if self._has_key(stripped_name): + node.left.type = self._lookup(stripped_name) + elif hasattr(value, 'get_type'): + node.left.type = value.get_type(self) + elif hasattr(value, 'type'): + node.left.type = value.type + elif isinstance(value, C.FunctionCall): + if self._has_key(value.func): + node.left.type = self._lookup(value.func) + elif hasattr(value, 'get_type'): + node.left.type = value.get_type(self) + elif hasattr(value, 'type'): + node.left.type = value.type + elif isinstance(value, C.String): + node.left.type = ct.c_char_p() + elif isinstance(value, C.SymbolRef): + node.left.type = self._lookup(value.name) + + self.__add_entry(node.left.name, node.left.type) + return node diff --git a/ctree/tune.py b/ctree/tune.py index 40fa5b6..f25ad23 100644 --- a/ctree/tune.py +++ b/ctree/tune.py @@ -29,18 +29,19 @@ def report(self, **kwargs): pass -class NullTuningDriver(TuningDriver): +class ConstantTuningDriver(TuningDriver): """ - Provides a stream of None's, and ignores reports()s. + Provides a stream of the same config, and ignores reports()s. """ - def __init__(self): + def __init__(self, config=None): """Do nothing.""" - super(NullTuningDriver, self).__init__() + super(ConstantTuningDriver, self).__init__() + self._config = config def _get_configs(self): """Yield the empty configuration.""" while True: - yield {} + yield self._config def report(self, *args, **kwargs): """Ignore reports.""" @@ -53,6 +54,10 @@ class Parameter(object): def __init__(self, name): """Create a parameter with the given name.""" self.name = name + self._values = [] + + def values(self): + return self._values class IntegerParameter(Parameter): @@ -62,8 +67,46 @@ def __init__(self, name, lower_bound, upper_bound): super(IntegerParameter, self).__init__(name) self._values = range(lower_bound, upper_bound) - def values(self): - return self._values + +class BooleanParameter(Parameter): + """A boolean parameter.""" + def __init__(self, name): + """Create a bool parameter.""" + super(BooleanParameter, self).__init__(name) + self._values = [True, False] + + +class EnumParameter(Parameter): + """A enum parameter.""" + def __init__(self, name, values): + """Create an enum parameter.""" + super(EnumParameter, self).__init__(name) + self._values = values + + +class IntegerArrayParameter(Parameter): + """An array of integers.""" + def __init__(self, name, count=1, lower_bound=0, upper_bound=1): + """Create an IntArray parameter.""" + super(IntegerArrayParameter, self).__init__(name) + self._values = itertools.product(range(lower_bound,upper_bound), repeat=count) + + +class EnumArrayParameter(Parameter): + """An array of enums.""" + def __init__(self, name, count=1, values=None): + """Create an EnumArray parameter.""" + super(EnumArrayParameter, self).__init__(name) + values = values if values else [] + self._values = itertools.product(values, repeat=count) + + +class BooleanArrayParameter(Parameter): + """An array of booleans.""" + def __init__(self, name, count=1): + """Create an BooleanArray parameter.""" + super(BooleanArrayParameter, self).__init__(name) + self._values = itertools.product([True,False], repeat=count) class Result(object): @@ -84,7 +127,7 @@ def __init__(self, time = float('inf'), class Objective(object): def compare(self, result0, result1): - raise NotImplementedException() + raise NotImplementedError() class MinimizeTime(Objective): diff --git a/ctree/types.py b/ctree/types.py index b82a314..7dc399a 100644 --- a/ctree/types.py +++ b/ctree/types.py @@ -1,44 +1,170 @@ -import abc +from __future__ import absolute_import -from ctree.nodes import CtreeNode -from ctree.visitors import NodeVisitor +import types +import sys +import ctypes +from ctypes import * -class CtreeType(CtreeNode): - def codegen(self, indent=0): - raise Exception("%s should override codegen()" % type(self)) +import logging - def as_ctypes(self): - raise Exception("%s should override as_ctypes()" % type(self)) +from ctree import _TYPE_CODEGENERATORS as generators +from ctree import _TYPE_RECOGNIZERS as recognizers - def __eq__(self, other): - return str(self) == str(other) +log = logging.getLogger(__name__) - def __hash__(self): - return hash(str(self)) +def register_type_codegenerators(codegen_dict): + """ + Registers routines for generating code for types. + + :param codegen_dict: Maps type classes to functions that + take an instance of that class and return the corresponding + string. + """ + if sys.version_info >= (3, 0): + existing_keys = generators.keys() + new_keys = codegen_dict.keys() + else: + existing_keys = generators.viewkeys() + new_keys = codegen_dict.viewkeys() + intersection = existing_keys & new_keys + if intersection: + log.warning("replacing existing type_codegenerator for %s", + intersection) + + for genfn in generators.values(): + assert callable(genfn), "Found a non-callable type_codegen: %s" % genfn + + generators.update(codegen_dict) + + +def register_type_recognizers(typerec_dict): + """ + Registers routines for getting ctypes objects from Python objects. + + :param typerec_dict: Maps Python classes to functions that + take an instance of that class and return the corresponding + ctypes object. + """ + if sys.version_info >= (3, 0): + existing_keys = recognizers.keys() + new_keys = typerec_dict.keys() + else: + existing_keys = recognizers.viewkeys() + new_keys = typerec_dict.viewkeys() + intersection = existing_keys & new_keys + if intersection: + log.warning("replacing existing type_recognizer for %s", intersection) + + for genfn in recognizers.values(): + assert callable(genfn), "Found a non-callable type_codegen: %s" % genfn -class TypeFetcher(NodeVisitor): + recognizers.update(typerec_dict) + + +def get_ctype(py_obj): """ - Dynamically computes the type of the Expression. + Given a python object, this routine tries to return the + closest ctype type instance corresponding to that object. + + :param py_obj: A python object. """ - pass + bases = [type(py_obj)] + while bases: + base = bases.pop() + bases += base.__bases__ + try: + return recognizers[base](py_obj) + except KeyError: + pass + raise ValueError("No type recognizer defined for %s." % type(py_obj)) +def get_c_type_from_numpy_dtype(dtype_specified): + """ + Get the ctype corresponding to a given numpy.dtype + :param: dtype_specified - the numpy.dtype + :return: the ctype corresponding the the dtype_specified. None unable to match + """ + typemap = {} + for t in (c_byte, c_short, c_int, c_long, c_longlong): + typemap["i%s" % sizeof(t)] = t.__ctype_be__ + for t in (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong): + typemap["u%s" % sizeof(t)] = t.__ctype_be__ + for t in (c_float, c_double): + typemap["f%s" % sizeof(t)] = t.__ctype_be__ + typemap["|b1"] = c_bool + typemap["|i1"] = c_byte + typemap["|u1"] = c_ubyte -class CtreeTypeResolver(object): - __metaclass__ = abc.ABCMeta + if dtype_specified.descr[0][1] in typemap: + return typemap[dtype_specified.descr[0][1]] + else: + return None - @staticmethod - @abc.abstractmethod - def resolve(obj): - pass +def codegen_type(ctype): + """ + Unparses the given ctype. + + :param ctype: A ctype type instance to be unparsed. + """ + assert not isinstance(ctype, type), \ + "Expected a ctypes type instance, not %s, (%s):" % (ctype, type(ctype)) + bases = [type(ctype)] + while bases: + base = bases.pop() + bases.extend(base.__bases__) + try: + val = generators[base](ctype) + return val + except KeyError: + pass + raise ValueError("No code generator defined for %s." % type(ctype)) -def get_ctree_type(obj): - from ctree.c.types import CTypeResolver, NumpyTypeResolver +def get_suffix(ctype): + ctype = ctype if isinstance(ctype, type) else type(ctype) + size = ctypes.sizeof(ctype) + suffix = "" + if size >= ctypes.sizeof(ctypes.c_ulong): + suffix += "l" + if size > ctypes.sizeof(ctypes.c_ulonglong): + suffix += "l" + if ctype(-1).value > 0: # if it becomes positive it indicates unsigned + suffix += "u" + return suffix + + +def get_common_ctype(ctypes_list): + """ + :param ctypes_list: iterable of ctypes + :return: calculates the proper ctype for coercion of all types, as per + + If either is long double the other is promoted to long double + If either is double the other is promoted to double + If either is float the other is promoted to float + If either is long long unsigned int the other is promoted to long long unsigned int + If either is long long int the other is promoted to long long int + If either is long unsigned int the other is promoted to long unsigned int + If either is long int the other is promoted to long int + if either is unsigned int the other is promoted to unsigned int + If either is int the other is promoted to int + Both operands are promoted to int + """ - for resolver in [CTypeResolver(), NumpyTypeResolver()]: - ty = resolver.resolve(obj) - if ty is not None: - return ty - raise Exception("Unable to resolve type for %s." % repr(obj)) + # lowest ranking takes precedence + rankings = [ctypes.c_longdouble, ctypes.c_double, ctypes.c_float, + ctypes.c_long, ctypes.c_uint, ctypes.c_int, ctypes.c_byte, + ctypes.c_wchar, ctypes.c_char, ctypes.c_bool, ctypes.c_void_p] + filtered = [] + for c_type in ctypes_list: + if c_type not in rankings: + return c_type + filtered.append(c_type) + if filtered: + return min(filtered, key=rankings.index) + else: + return ctypes.c_void_p diff --git a/ctree/util.py b/ctree/util.py index b981d61..2d5e30f 100644 --- a/ctree/util.py +++ b/ctree/util.py @@ -5,6 +5,10 @@ from textwrap import dedent import ctree +import time + +import functools +import operator def singleton(cls): @@ -13,6 +17,14 @@ def singleton(cls): return instance +def product(nums): + return functools.reduce(operator.mul, nums, 1) + + +def strides(shape): + return [product(shape[x:]) for x in range(1, len(shape))] + [1] + + def truncate(text): max_display_lines = ctree.CONFIG.getint("log", "max_lines_per_source") n_lines = len(text.splitlines()) @@ -32,14 +44,18 @@ def lower_case_underscore_to_camel_case(string): return class_.join('', map(class_.capitalize, string.split('_'))) -def flatten(obj_or_list): +def flatten(obj): """Iterator for all objects arbitrarily nested in lists.""" - if isinstance(obj_or_list, list): - for gen in map(flatten, obj_or_list): + if isinstance(obj, (set, list)): + for gen in map(flatten, obj): + for elem in gen: + yield elem + elif isinstance(obj, (dict)): + for gen in map(flatten, obj.itervalues()): for elem in gen: yield elem else: - yield obj_or_list + yield obj def enumerate_flatten(obj_or_list): @@ -56,17 +72,30 @@ def highlight(code, language='c'): """Syntax-highlight code using pygments, if installed.""" try: from pygments.formatters.terminal256 import Terminal256Formatter - from pygments.lexers.compiled import CLexer - from pygments.lexers.asm import LlvmLexer from pygments import highlight except ImportError: log.info("install pygments for syntax-highlighted output.") return code - if language.lower() == 'llvm': lexer = LlvmLexer() - elif language.lower() == 'c': lexer = CLexer() + if language.lower() == 'llvm': + from pygments.lexers.asm import LlvmLexer as TheLexer + elif language.lower() == 'c': + from pygments.lexers.compiled import CLexer as TheLexer + elif language.lower() == 'diff': + from pygments.lexers.text import DiffLexer as TheLexer + elif language.lower() == 'ini': + from pygments.lexers.text import IniLexer as TheLexer else: raise ValueError("Unrecognized highlight language: %s" % language) style = ctree.CONFIG.get('log', 'pygments_style') - return highlight(code, lexer, Terminal256Formatter(style=style)) + return highlight(code, TheLexer(), Terminal256Formatter(style=style)) + + +class Timer: # pragma: no cover + def __enter__(self): + self.start = time.clock() + return self + + def __exit__(self, *args): + self.interval = time.clock() - self.start diff --git a/ctree/visual/dot_manager.py b/ctree/visual/dot_manager.py index d29c18f..909d69b 100644 --- a/ctree/visual/dot_manager.py +++ b/ctree/visual/dot_manager.py @@ -1,5 +1,8 @@ +import tempfile + __author__ = 'Chick Markley' + class DotManager(object): """ take ast and return an ipython image file @@ -7,25 +10,33 @@ class DotManager(object): @staticmethod def dot_ast_to_image(ast_node): - from ctree.dotgen import to_dot - - dot_text = to_dot(ast_node) - + dot_text = ast_node.to_dot() return DotManager.dot_text_to_image(dot_text) @staticmethod - def dot_ast_to_browser(ast_node, file_name): - from ctree.dotgen import to_dot - - dot_text = to_dot(ast_node) + def dot_ast_to_browser(ast_node, file_name=None): + dot_text = ast_node.to_dot() dot_output = DotManager.run_dot(dot_text) - with open(file_name, "wb") as f: - f.write(dot_output) + if file_name is None: + with tempfile.NamedTemporaryFile(mode='wb', suffix=".png", delete=False) as f: + f.write(dot_output) + file_name = f.name + else: + with open(file_name, "wb") as f: + f.write(dot_output) import subprocess subprocess.check_output(["open", file_name]) + @staticmethod + def dot_ast_to_file(ast_node, file_name): + dot_text = ast_node.to_dot() + dot_output = DotManager.run_dot(dot_text) + + with open(file_name, "wb") as f: + f.write(dot_output) + @staticmethod def dot_text_to_image(text): try: @@ -41,7 +52,16 @@ def run_dot(code, options=None, output_format='png', file_name=None): # mostly copied from sphinx.ext.graphviz.render_dot import os from subprocess import Popen, PIPE - from sphinx.util.osutil import EPIPE, EINVAL + try: + from sphinx.util.osutil import EPIPE, EINVAL + except ImportError: + EPIPE, EINVAL = None, None + raise RuntimeError( + """ + graphical display of ASTs requires that the sphinx + package be installed. + """ + ) if not options: options = [] @@ -49,15 +69,25 @@ def run_dot(code, options=None, output_format='png', file_name=None): if file_name: dot_args += ['>',file_name] - if os.name == 'nt': - # Avoid opening shell window. - # * https://github.com/tkf/ipython-hierarchymagic/issues/1 - # * http://stackoverflow.com/a/2935727/727827 - p = Popen(dot_args, stdout=PIPE, stdin=PIPE, stderr=PIPE, - creationflags=0x08000000) - else: - p = Popen(dot_args, stdout=PIPE, stdin=PIPE, stderr=PIPE) + p = None + try: + if os.name == 'nt': + # Avoid opening shell window. + # * https://github.com/tkf/ipython-hierarchymagic/issues/1 + # * http://stackoverflow.com/a/2935727/727827 + p = Popen(dot_args, stdout=PIPE, stdin=PIPE, stderr=PIPE, + creationflags=0x08000000) + else: + p = Popen(dot_args, stdout=PIPE, stdin=PIPE, stderr=PIPE) + except OSError: + raise RuntimeError( + """ + Attempting to generate AST image. most likely dot (available through graphviz) is not installed + or is not in your path + """ + ) went_wrong = False + stdout, stderr = None, None # for PEP-8 try: # Graphviz may close standard input when an error occurs, # resulting in a broken pipe on communicate() diff --git a/doc/ctree.metrics.rst b/doc/ctree.metrics.rst new file mode 100644 index 0000000..945592d --- /dev/null +++ b/doc/ctree.metrics.rst @@ -0,0 +1,22 @@ +ctree.metrics package +===================== + +Submodules +---------- + +ctree.metrics.watts_up_reader module +------------------------------------ + +.. automodule:: ctree.metrics.watts_up_reader + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: ctree.metrics + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.np.rst b/doc/ctree.np.rst new file mode 100644 index 0000000..74854c8 --- /dev/null +++ b/doc/ctree.np.rst @@ -0,0 +1,10 @@ +ctree.np package +================ + +Module contents +--------------- + +.. automodule:: ctree.np + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.opentuner.rst b/doc/ctree.opentuner.rst new file mode 100644 index 0000000..fa06191 --- /dev/null +++ b/doc/ctree.opentuner.rst @@ -0,0 +1,22 @@ +ctree.opentuner package +======================= + +Submodules +---------- + +ctree.opentuner.driver module +----------------------------- + +.. automodule:: ctree.opentuner.driver + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: ctree.opentuner + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.sse.rst b/doc/ctree.sse.rst deleted file mode 100644 index 30275e6..0000000 --- a/doc/ctree.sse.rst +++ /dev/null @@ -1,38 +0,0 @@ -ctree.sse package -================= - -Submodules ----------- - -ctree.sse.codegen module ------------------------- - -.. automodule:: ctree.sse.codegen - :members: - :undoc-members: - :show-inheritance: - -ctree.sse.dotgen module ------------------------ - -.. automodule:: ctree.sse.dotgen - :members: - :undoc-members: - :show-inheritance: - -ctree.sse.nodes module ----------------------- - -.. automodule:: ctree.sse.nodes - :members: - :undoc-members: - :show-inheritance: - - -Module contents ---------------- - -.. automodule:: ctree.sse - :members: - :undoc-members: - :show-inheritance: diff --git a/doc/ctree.templates.rst b/doc/ctree.templates.rst new file mode 100644 index 0000000..93d7167 --- /dev/null +++ b/doc/ctree.templates.rst @@ -0,0 +1,38 @@ +ctree.templates package +======================= + +Submodules +---------- + +ctree.templates.codegen module +------------------------------ + +.. automodule:: ctree.templates.codegen + :members: + :undoc-members: + :show-inheritance: + +ctree.templates.dotgen module +----------------------------- + +.. automodule:: ctree.templates.dotgen + :members: + :undoc-members: + :show-inheritance: + +ctree.templates.nodes module +---------------------------- + +.. automodule:: ctree.templates.nodes + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: ctree.templates + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.tools.generators.rst b/doc/ctree.tools.generators.rst new file mode 100644 index 0000000..0260a5a --- /dev/null +++ b/doc/ctree.tools.generators.rst @@ -0,0 +1,29 @@ +ctree.tools.generators package +============================== + +Subpackages +----------- + +.. toctree:: + + ctree.tools.generators.templates + +Submodules +---------- + +ctree.tools.generators.builder module +------------------------------------- + +.. automodule:: ctree.tools.generators.builder + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: ctree.tools.generators + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.tools.generators.templates.create.rst b/doc/ctree.tools.generators.templates.create.rst new file mode 100644 index 0000000..7bce4c5 --- /dev/null +++ b/doc/ctree.tools.generators.templates.create.rst @@ -0,0 +1,17 @@ +ctree.tools.generators.templates.create package +=============================================== + +Subpackages +----------- + +.. toctree:: + + ctree.tools.generators.templates.create.specializer_package + +Module contents +--------------- + +.. automodule:: ctree.tools.generators.templates.create + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.tools.generators.templates.create.specializer_package.rst b/doc/ctree.tools.generators.templates.create.specializer_package.rst new file mode 100644 index 0000000..dc101bd --- /dev/null +++ b/doc/ctree.tools.generators.templates.create.specializer_package.rst @@ -0,0 +1,10 @@ +ctree.tools.generators.templates.create.specializer_package package +=================================================================== + +Module contents +--------------- + +.. automodule:: ctree.tools.generators.templates.create.specializer_package + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.tools.generators.templates.rst b/doc/ctree.tools.generators.templates.rst new file mode 100644 index 0000000..76b66ea --- /dev/null +++ b/doc/ctree.tools.generators.templates.rst @@ -0,0 +1,17 @@ +ctree.tools.generators.templates package +======================================== + +Subpackages +----------- + +.. toctree:: + + ctree.tools.generators.templates.create + +Module contents +--------------- + +.. automodule:: ctree.tools.generators.templates + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.tools.rst b/doc/ctree.tools.rst new file mode 100644 index 0000000..591fb02 --- /dev/null +++ b/doc/ctree.tools.rst @@ -0,0 +1,29 @@ +ctree.tools package +=================== + +Subpackages +----------- + +.. toctree:: + + ctree.tools.generators + +Submodules +---------- + +ctree.tools.runner module +------------------------- + +.. automodule:: ctree.tools.runner + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: ctree.tools + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/ctree.visual.rst b/doc/ctree.visual.rst new file mode 100644 index 0000000..370268e --- /dev/null +++ b/doc/ctree.visual.rst @@ -0,0 +1,22 @@ +ctree.visual package +==================== + +Submodules +---------- + +ctree.visual.dot_manager module +------------------------------- + +.. automodule:: ctree.visual.dot_manager + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: ctree.visual + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/devtips.rst b/doc/devtips.rst index fcb1e43..05ee46d 100644 --- a/doc/devtips.rst +++ b/doc/devtips.rst @@ -47,3 +47,17 @@ To switch back to your default python installation, run:: You can re-activate the virtualenv at any time using:: source venv-2.7/bin/activate + + +Cleaning the Folder of Temporary Files +-------------------------------------- + +If ``ctree`` is exiting uncleanly it may leave compilation directories in the temporary folder. If there are lots of them, ``rm`` may not be sufficient to remove them:: + + $ CTREE_TMP_DIR=/var/folders/k3/_z9txmtx3vd1t_64hbx9y4qr0000gn/T + $ rm -rf $CTREE_TMP_DIR/ctree-* + zsh: argument list too long: rm + +Use a command like the following to rememdy the situation:: + + $ find $CTREE_TMP_DIR -name "ctree-*" | xargs rm -rf diff --git a/doc/modules.rst b/doc/modules.rst new file mode 100644 index 0000000..d1fe7a5 --- /dev/null +++ b/doc/modules.rst @@ -0,0 +1,7 @@ +ctree +===== + +.. toctree:: + :maxdepth: 4 + + ctree diff --git a/doc/openmp.rst b/doc/openmp.rst index e6eb494..cfb8c46 100644 --- a/doc/openmp.rst +++ b/doc/openmp.rst @@ -6,6 +6,13 @@ Using OpenMP with ctree ``ctree`` can be configured to use `Intel's version of Clang `_ with support for OpenMP. +Dependencies +============ + +On OSX, ensure that you have the XCode Command Line Tools installed:: + + xcode-select --install + Installing OpenMP/Clang ------------------ @@ -20,44 +27,80 @@ First, get the source code:: $ git clone https://github.com/clang-omp/compiler-rt llvm/projects/compiler-rt $ git clone -b clang-omp https://github.com/clang-omp/clang llvm/tools/clang +IMPORTANT: At this point you need to decide where you want to install your llvm. For this example +We will assume that it will be in /usr/local/llvm_build, for the next step do this from the command +line, via:: + + $ export LLVM_BUILD_PATH=/usr/local/llvm_build # Change this for your particular environment + $ mkdir $LLVM_BUILD_PATH + +Depending on the location of LLVM_BUILD_PATH it may be necessary to use sudo with the mkdir command above, and +in that case you may also want to make that directory owned by someone other than root + Now, build clang/llvm:: $ mkdir build $ cd build - $ ../llvm/configure --enable-optimized --prefix=YOUR_INSTALL_PATH # i.e. /opt/llvm-omp + $ ../llvm/configure --enable-optimized --prefix=$LLVM_BUILD_PATH $ REQUIRES_RTTI=1 make $ make install -Setup your environment variables in your shell configuration. On Mac OS X, -replace LD_LIBRARY_PATH with DYLD_LIBRARY_PATH.:: - - PATH=/install/prefix/bin:$PATH - C_INCLUDE_PATH=/install/prefix/include::$C_INCLUDE_PATH - CPLUS_INCLUDE_PATH=/install/prefix/include::$CPLUS_INCLUDE_PATH - LIBRARY_PATH=/install/prefix/lib::$LIBRARY_PATH - LD_LIBRARY_PATH=/install/prefix/lib::$LD_LIBRARY_PATH - Download and install the Intel OpenMP Runtime Library from `https://www.openmprtl.org/`, or by installing the `Intel Compilers -`_. +`_. You can use the evaluation version of the Intel compilers which will install the OpenMP runtime library. After 30 days the compilers will cease to work but the runtime library will still be usable. -Include the OpenMP RTL, for OSX with the evaluation Intel Compilers you can do:: +Setup your environment variables for a linux system (add this to your .bashrc or .zshrc) in your shell configuration.:: + export LLVM_BUILD_PATH=/usr/local/llvm_build # Change this for your particular environment + export OPENMP_RUNTIME_PATH=/usr/local/open_mp # Change this + + export LLVM_BUILD_PATH=/usr/local/llvm-omp + export OPENMP_RUNTIME_PATH=/opt/intel/composerxe + + export LD_LIBRARY_PATH=/opt/intel/composerxe/lib + + export CPLUS_INCLUDE_PATH=/usr/include/:/usr/include/c++ + export C_INCLUDE_PATH=/usr/include/:/usr/include/c++ + + export PATH=$LLVM_BUILD_PATH/bin:$PATH + export C_INCLUDE_PATH=$LLVM_BUILD_PATH/include:$OPENMP_RUNTIME_PATH/include:$C_INCLUDE_PATH + export CPLUS_INCLUDE_PATH=$LLVM_BUILD_PATH/include:$OPENMP_RUNTIME_PATH/include:$CPLUS_INCLUDE_PATH + export LIBRARY_PATH=$LLVM_BUILD_PATH/lib::$LIBRARY_PATH + export LD_LIBRARY_PATH=$LLVM_BUILD_PATH/lib::$LD_LIBRARY_PATH + +On Mac OS X, replace LD_LIBRARY_PATH with DYLD_LIBRARY_PATH.:: + + export LLVM_BUILD_PATH=/usr/local/llvm_build # Change this for your particular environment + export OPENMP_RUNTIME_PATH=/usr/local/open_mp # Change this + + export LLVM_BUILD_PATH=/usr/local/llvm-omp + export OPENMP_RUNTIME_PATH=/opt/intel/composerxe + + export LD_LIBRARY_PATH=/opt/intel/composerxe/lib + + export CPLUS_INCLUDE_PATH=/usr/include/:/usr/include/c++ + export C_INCLUDE_PATH=/usr/include/:/usr/include/c++ + + export PATH=$LLVM_BUILD_PATH/bin:$PATH + export C_INCLUDE_PATH=$LLVM_BUILD_PATH/include:$OPENMP_RUNTIME_PATH/include:$C_INCLUDE_PATH + export CPLUS_INCLUDE_PATH=$LLVM_BUILD_PATH/include:$OPENMP_RUNTIME_PATH/include:$CPLUS_INCLUDE_PATH + export LIBRARY_PATH=$LLVM_BUILD_PATH/lib::$LIBRARY_PATH + export LD_LIBRARY_PATH=$LLVM_BUILD_PATH/lib::$LD_LIBRARY_PATH - DYLD_LIBRARY_PATH=/opt/intel/composerxe/lib:$DYLD_LIBRARY_PATH + export DYLD_LIBRARY_PATH=/opt/intel/composerxe/lib:$DYLD_LIBRARY_PATH Download and checkout gentoo90's llvmpy branch with llvm-3.4 support and build it:: - $ git clone -b llvm-3.4 https://github.com/gentoo90/llvmpy.git + $ git clone -b llvm-3.4 http://github.com/llvmpy/llvmpy.git $ cd llvmpy $ LLVM_CONFIG_PATH=YOUR_INSTALL_PATH/bin/llvm-config python setup.py install Update your ~/.ctree.cfg to use the proper clang and the openmp RTL:: - [jit] + [omp] CC = /Users/your_name/opt/llvm-omp-3.4/bin/clang CFLAGS = -march=native -O3 -fopenmp -I/opt/intel/composerxe/include diff --git a/examples/ArrayDoubler.py b/examples/ArrayDoubler.py index cc70c71..2a83a24 100644 --- a/examples/ArrayDoubler.py +++ b/examples/ArrayDoubler.py @@ -2,19 +2,18 @@ Parses the python AST below, transforms it to C, JITs it, and runs it. """ -import logging +#logging.basicConfig(level=10) -logging.basicConfig(level=20) +import ctypes as ct import numpy as np -from ctree.frontend import get_ast from ctree.c.nodes import * -from ctree.c.types import * -from ctree.dotgen import to_dot +from ctree.nodes import Project from ctree.transformations import * from ctree.jit import LazySpecializedFunction -from ctree.types import get_ctree_type +from ctree.jit import ConcreteSpecializedFunction +# from ctypes import CFUNCTYPE # --------------------------------------------------------------------------- # Specializer code @@ -29,10 +28,7 @@ def args_to_subconfig(self, args): """ A = args[0] return { - 'A_len': len(A), - 'A_dtype': A.dtype, - 'A_ndim': A.ndim, - 'A_shape': A.shape, + 'ptr': np.ctypeslib.ndpointer(A.dtype, A.ndim, A.shape), } def transform(self, py_ast, program_config): @@ -41,26 +37,22 @@ def transform(self, py_ast, program_config): given in program_config. """ arg_config, tuner_config = program_config - len_A = arg_config['A_len'] - A_dtype = arg_config['A_dtype'] - A_ndim = arg_config['A_ndim'] - A_shape = arg_config['A_shape'] - - inner_type = get_ctree_type(A_dtype) - array_type = NdPointer(A_dtype, A_ndim, A_shape) - apply_one_typesig = FuncType(inner_type, [inner_type]) + array_type = arg_config['ptr'] + nItems = np.prod(array_type._shape_) + inner_type = array_type._dtype_.type() + kernel_func_name = 'apply' tree = CFile("generated", [ py_ast.body[0], - FunctionDecl(Void(), "apply_all", - params=[SymbolRef("A", array_type)], + FunctionDecl(None, "apply_all", + params=[SymbolRef("A", array_type())], defn=[ - For(Assign(SymbolRef("i", Int()), Constant(0)), - Lt(SymbolRef("i"), Constant(len_A)), + For(Assign(SymbolRef("i", c_int()), Constant(0)), + Lt(SymbolRef("i"), Constant(nItems)), PostInc(SymbolRef("i")), [ Assign(ArrayRef(SymbolRef("A"), SymbolRef("i")), - FunctionCall(SymbolRef("apply"), [ArrayRef(SymbolRef("A"), + FunctionCall(SymbolRef(kernel_func_name), [ArrayRef(SymbolRef("A"), SymbolRef("i"))])), ]), ] @@ -69,46 +61,49 @@ def transform(self, py_ast, program_config): tree = PyBasicConversions().visit(tree) - apply_one = tree.find(FunctionDecl, name="apply") + apply_one = PyBasicConversions().visit(tree.body[0]) + apply_one.name = kernel_func_name apply_one.set_static().set_inline() - apply_one.set_typesig(apply_one_typesig) + apply_one.return_type = inner_type + apply_one.params[0].type = inner_type - entry_point_typesig = tree.find(FunctionDecl, name="apply_all").get_type().as_ctype() + c_doubler = CFile("generated", [tree]) + return [c_doubler] - return Project([tree]), entry_point_typesig + def finalize(self, transform_result, program_config): + c_doubler = transform_result[0] + proj = Project([c_doubler]) -class ArrayOp(object): - """ - A class for managing independent operation on elements - in numpy arrays. - """ + arg_config, tuner_config = program_config + array_type = arg_config['ptr'] + entry_type = ct.CFUNCTYPE(None, array_type) - def __init__(self): - """Instantiate translator.""" - self.c_apply_all = OpTranslator(get_ast(self.apply), "apply_all") + concrete_Fn = ArrayFn() + return concrete_Fn.finalize("apply_all", proj, entry_type) - def __call__(self, A): - """Apply the operator to the arguments via a generated function.""" - return self.c_apply_all(A) +class ArrayFn(ConcreteSpecializedFunction): + def finalize(self, entry_point_name, project_node, entry_typesig): + self._c_function = self._compile(entry_point_name, project_node, entry_typesig) + return self + def __call__(self, A): + return self._c_function(A) # --------------------------------------------------------------------------- # User code -class Doubler(ArrayOp): - """Double elements of the array.""" - - def apply(n): - return n * 2 - +def double(n): + return n * 2 def py_doubler(A): A *= 2 - def main(): - c_doubler = Doubler() + + # create a class called Doubler that has the function double(n) as an @staticmethod + c_doubler= OpTranslator.from_function(double, "Doubler") + # doubling doubles actual_d = np.ones(12, dtype=np.float64) @@ -142,4 +137,10 @@ def main(): if __name__ == '__main__': + # Testing conventional (non-lambda) kernel function implementation main() + + # Testing lambda kernel function implementation + double = lambda x: x * 2 + main() + diff --git a/examples/AstToDot.py b/examples/AstToDot.py index c91263a..0937c40 100644 --- a/examples/AstToDot.py +++ b/examples/AstToDot.py @@ -8,20 +8,18 @@ The resulting file can be viewed with a visualizer like Graphiz. """ +from ctypes import * from ctree.nodes import * from ctree.c.nodes import * -from ctree.c.types import * - -from ctree.dotgen import to_dot def main(): stmt0 = Assign(SymbolRef('foo'), Constant(123.4)) - stmt1 = FunctionDecl(Float(), SymbolRef("bar"), [ - SymbolRef("spam", Int()), SymbolRef("eggs", Long())], [String("baz")]) - stmt3 = [[SymbolRef("AAAAA")]] + stmt1 = FunctionDecl(c_float(), SymbolRef("bar"), [ + SymbolRef("spam", c_int()), SymbolRef("eggs", c_long())], [String("baz")]) + stmt3 = [[SymbolRef("abc")]] tree = CFile("myfile", [stmt0, stmt1, stmt3]) - print (to_dot(tree)) + print (tree.to_dot()) if __name__ == '__main__': diff --git a/examples/Distrib.py b/examples/Distrib.py new file mode 100644 index 0000000..f68676c --- /dev/null +++ b/examples/Distrib.py @@ -0,0 +1,740 @@ +""" +Code generator for the expression A*(B+C), where A, B, and C are vectors +and all operations are element-wise. +""" + +n = 0 + +import itertools +import logging +import copy + +logging.basicConfig(level=20) + +import numpy as np +import pycl as cl +from ctypes import * + +import ctree.np +import ctree.ocl + +from ctree.frontend import get_ast +from ctree.c.nodes import * +from ctree.cpp.nodes import * +from ctree.omp.macros import * +from ctree.ocl.macros import * +from ctree.templates.nodes import * +from ctree.transformations import * +from ctree.visitors import NodeVisitor +from ctree.jit import LazySpecializedFunction +from ctree.jit import ConcreteSpecializedFunction + + +# --------------------------------------------------------------------------- +# Specializer code - nodes + +class Vector(CtreeNode): + def __init__(self, name, type=None, loc=None): + self.name = name + self.loc = loc + self.type = type + self._loc_cache = {} + + def label(self): + return "name: %s\\nloc: %s\\ntype: %s" % \ + (self.name, self.loc, self.type) + + def get_type(self): + return self.type + + def codegen(self, indent=0): + return "%s %s" % (self.get_type(), self.name) + + def copy_to(self, mem): + if mem not in self._loc_cache: + self._loc_cache[mem] = CopiedVector(self, to=mem, type=self.type) + return self._loc_cache[mem] + +class CopiedVector(Vector): + _fields = ["data"] + _next_id = 0 + def __init__(self, data=None, to=None): + self.data = data + name = "copied%d" % self._next_id + CopiedVector._next_id += 1 + super(CopiedVector, self).__init__(name=name, loc=to) + + def label(self): + to = "to: %s" % self.loc + frm = "from: %s" % getattr(self.data, 'loc', '?') + ty = "type: %s" % getattr(self, 'type', '?') + return "name: %s\\n%s\\n%s\\n%s" % (self.name, to, frm, ty) + + +class ComputedVector(Vector): + _fields = ["data"] + _next_id = 0 + def __init__(self, data=None, name=None, loc=None): + self.data = data + if not name: + name = "computed%d" % self._next_id + ComputedVector._next_id += 1 + super(ComputedVector, self).__init__(name=name, loc=loc) + +# --------------------------------------------------------------------------- +# Specializer code - transformers + +class ApplyDistributiveProperty(NodeTransformer): + def __init__(self, directives): + super(ApplyDistributiveProperty, self).__init__() + self._directives = iter(directives) + + def visit_BinaryOp(self, node): + ab = node.left = self.visit(node.left) + cd = node.right = self.visit(node.right) + dist_left = isinstance(ab, BinaryOp) and isinstance(ab.op, Op.Add) + dist_right = isinstance(cd, BinaryOp) and isinstance(cd.op, Op.Add) + if isinstance(node.op, Op.Mul): + if dist_right and self._directives.next(): + c, d = cd.left, cd.right + abc = self.visit( Mul(ab,c) ) + abd = self.visit( Mul(ab,d) ) + return Add(abc, abd) + elif dist_left and self._directives.next(): + a, b = ab.left, ab.right + acd = self.visit( Mul(a, cd) ) + bcd = self.visit( Mul(b, cd) ) + return Add(acd, bcd) + return node + + +class ApplyAssociativeProperty(NodeTransformer): + _supported_ops = (Op.Add, Op.Mul, Op.BitAnd) + + def __init__(self, directives): + super(ApplyAssociativeProperty, self).__init__() + self._directives = iter(directives) + + def visit_BinaryOp(self, node): + l = node.left = self.visit(node.left) + r = node.right = self.visit(node.right) + + if isinstance(node.op, self._supported_ops): + assoc_right = isinstance(l, BinaryOp) and type(node.op) == type(l.op) + assoc_left = isinstance(r, BinaryOp) and type(node.op) == type(r.op) + if assoc_right and self._directives.next(): + ll, lr = l.left, l.right + return BinaryOp(ll, node.op, BinaryOp(lr, node.op, r)) + if assoc_left and self._directives.next(): + rl, rr = r.left, r.right + return BinaryOp(BinaryOp(l, node.op, rl), node.op, rr) + return node + + +class VectorFinder(NodeTransformer): + def __init__(self, types, main_memory): + self._cache = {} + self._types = (ty() for ty in types) + self._main_memory = main_memory + + def visit_SymbolRef(self, node): + return self._cache[node.name] + + def visit_FunctionDecl(self, node): + for param in node.params: + self._cache[param.name] = Vector(param.name, self._types.next(), loc=self._main_memory) + return self.generic_visit(node) + + +class InsertIntermediates(NodeTransformer): + def __init__(self, main_memory): + self._main_memory = main_memory + + def visit_BinaryOp(self, node): + tree = self.generic_visit(node) + return ComputedVector(tree, loc=node.loc) + + def visit_Return(self, node): + answer = self.visit(node.value) + + if answer.loc != self._main_memory: + answer = CopiedVector(data=answer, to=self._main_memory) + + answer.name = "answer" + return answer + + class AssertHasAllIntermediates(NodeVisitor): + def visit_BinaryOp(self, node): + assert not isinstance(node.left, BinaryOp) + assert not isinstance(node.right, BinaryOp) + self.generic_visit(node) + + def visit(self, node): + proj = super(InsertIntermediates, self).visit(node) + InsertIntermediates.AssertHasAllIntermediates().visit(proj) + return proj + +BinaryOp.label = lambda self: "loc: %s" % getattr(self, 'loc', '?') + +class LocationTagger(NodeTransformer): + def __init__(self, locs): + self._locs = iter(locs) + + def visit_BinaryOp(self, node): + node.loc = self._locs.next() + return self.generic_visit(node) + + +class DoFusion(NodeTransformer): + def __init__(self, directives): + self._directives = iter(directives) + + def visit_BinaryOp(self, node): + tree = self.generic_visit(node) + if isinstance(tree.left, ComputedVector) and self._directives.next(): + tree.left = tree.left.data + if isinstance(tree.right, ComputedVector) and self._directives.next(): + tree.right = tree.right.data + return tree + + +class CopyInserter(NodeTransformer): + def __init__(self, main_memory): + self._locs = [main_memory] + self._copies = dict() + + def visit_CopiedVector(self, node): + self._locs.append(node.data.loc) + node = self.generic_visit(node) + self._locs.pop() + return node + + def visit_ComputedVector(self, node): + outer_loc = self._locs[-1] + self._locs.append(node.loc) + self.generic_visit(node) + if node.loc != outer_loc: + if node not in self._copies: + self._copies[node] = CopiedVector(data=node, to=outer_loc) + node = self._copies[node] + self._locs.pop() + return node + + def visit_Vector(self, node): + outer_loc = self._locs[-1] + self._locs.append(node.loc) + self.generic_visit(node) + if node.loc != outer_loc: + if node not in self._copies: + self._copies[node] = CopiedVector(data=node, to=outer_loc) + node = self._copies[node] + self._locs.pop() + return node + + +class AllocateIntermediates(NodeTransformer): + def __init__(self, dtype, length): + self.dtype = dtype + self.length = length + + def visit_ComputedVector(self, node): + node.mem, sym = node.loc.allocate(self.length, self.dtype, node.name) + node.lift(params=[(sym, node.mem)]) + node.type = sym.type + return self.generic_visit(node) + + def visit_CopiedVector(self, node): + node.mem, sym = node.loc.allocate(self.length, self.dtype, node.name) + node.lift(params=[(sym, node.mem)]) + node.type = sym.type + return self.generic_visit(node) + + +class GetWorkItems(NodeVisitor): + def visit_BinaryOp(self, node): + lhs = self.visit(node.left) + rhs = self.visit(node.right) + return lhs + rhs + + def visit_ComputedVector(self, node): + return [node] + +class FindParallelism(NodeVisitor): + def __init__(self, parallelize_directives): + super(FindParallelism, self).__init__() + self._parallelize = iter(parallelize_directives) + self._visited = [set()] + + def scope(self): + return self + + def in_scope(self, obj): + for scope in self._visited: + if obj in scope: + return True + return False + + def __enter__(self): + self._visited.append(set()) + + def __exit__(self, *args): + self._visited.pop() + + def visit_BinaryOp(self, node): + left = self.visit(node.left) + right = self.visit(node.right) + if left and right: + if self._parallelize.next(): + return frozenset([left, right]) + else: + return tuple([left, right]) + return left or right or None + + def visit_ComputedVector(self, node): + compute = self.visit(node.data) + if compute: + return (compute, node) + else: + return node + + def visit_CopiedVector(self, node): + copyin = self.visit(node.data) + if copyin: + return (copyin, node) + else: + return node + + def visit_FunctionDecl(self, node): + return tuple(self.visit(stmt) for stmt in node.defn) + + +class RefConverter(NodeTransformer): + class ToParamDecl(NodeTransformer): + def visit_Vector(self, node): + return SymbolRef(node.name, node.type) + def visit_ComputedVector(self, node): + return SymbolRef(node.name, node.type) + def visit_CopiedVector(self, node): + return SymbolRef(node.name, node.type) + + def visit_BinaryOp(self, node): + if isinstance(node.left, Vector): + node.left = ArrayRef(SymbolRef(node.left.name), SymbolRef("i")) + if isinstance(node.right, Vector): + node.right = ArrayRef(SymbolRef(node.right.name), SymbolRef("i")) + return self.generic_visit(node) + + def visit_FunctionDecl(self, node): + param_conv = RefConverter.ToParamDecl() + node.params = [param_conv.visit(p) for p in node.params] + node.defn = [self.visit(stmt) for stmt in node.defn] + return node + + +class KernelCall(CtreeNode): + _fields = ['args', 'kernel', 'queue'] + def __init__(self, location=None, name=None, global_size=0, local_size=0, args=None, kernel=None): + self.location = location + self.name = name + self.global_size = global_size + self.local_size = local_size + self.args = args + self.kernel = kernel + + if isinstance(self.local_size, int): + self.local_size = Constant(self.local_size) + if isinstance(self.global_size, int): + self.global_size = Constant(self.global_size) + + def label(self): + return "name: %s" % self.name + +class LowerKernelCalls(NodeTransformer): + def visit_KernelCall(self, node): + args = [] + for i, arg in enumerate(node.args): + size = SizeOf(SymbolRef(arg.name)) + setter = clSetKernelArg(node.name, i, size, Ref(SymbolRef(arg.name))) + setter.lift(params=arg._lift_params) + args.append(setter) + + kernel_decl = SymbolRef(node.name, cl.cl_kernel()) + kernel_symbol = kernel_decl.copy() + call = clEnqueueNDRangeKernel(node.location.symbol.copy(), kernel_symbol, work_dim=Constant(1), global_size=node.global_size, local_size=node.local_size) + + kernel = RefConverter().visit(node.kernel) + for param in kernel.params: + param.type = param.type.ptr_type + kernel.defn.insert(0, Assign(SymbolRef("i", c_int()), get_global_id(0))) + kernel_src = kernel.codegen() + call.body.append(CppComment(kernel_src)) + + context = node.location.queue.context + kernel_ptr = cl.clCreateProgramWithSource(context, kernel_src).build()[node.name] + call.lift(params=[(kernel_decl, kernel_ptr)]) + + return args + [call] + +def outline(tree, name="outlined"): + class VecGatherer(NodeTransformer): + def __init__(self): + self.signature = [] + + def visit_ComputedVector(self, node): + if node not in self.signature: + self.signature.append(node) + return node + + def visit_CopiedVector(self, node): + if node not in self.signature: + self.signature.append(node) + return node + + vec_gatherer = VecGatherer() + tree = vec_gatherer.visit(tree) + signature = vec_gatherer.signature + + if not isinstance(tree, list): + tree = [tree] + + return FunctionDecl(None, name, signature, tree) + + +class KernelOutliner(NodeTransformer): + def __init__(self, work_items): + self.work_items = work_items + + def visit_ComputedVector(self, node): + if isinstance(node.loc, OclMemory): + fn = outline(Assign(node, node.data), "outline_%s" % node.name) + call = KernelCall(node.loc, fn.name, self.work_items, 1, fn.params, fn.set_kernel()) + return call + else: + return node + + def visit_CopiedVector(self, node): + return node + + +class LowerLoopsAndCopies(NodeTransformer): + def __init__(self, nElems): + self.nElems = nElems + + def visit_ComputedVector(self, node): + i = SymbolRef("i", c_int()) + for_stmt = For(Assign(i, Constant(0)), + Lt(i.copy(), Constant(self.nElems)), + PostInc(i.copy()), [ + Assign( ArrayRef(SymbolRef(node.name), i.copy()), + self.visit(node.data) ) + ]) + + for_stmt._lift_params = node._lift_params + + return [CppComment("on %s" % node.loc), for_stmt] + + def visit_CopiedVector(self, node): + dst = node + src = node.data + + if isinstance(dst.loc, OclMemory) and \ + isinstance(src.loc, OclMemory): # device to device + params = [ + (src.loc.symbol, src.loc.queue), + (dst.loc.symbol, dst.loc.queue), + ] + queue_sym = dst.loc.symbol + call = clEnqueueCopyBuffer(queue_sym.copy(), + src.name, dst.name, 0, 0, dst.type.size) + elif isinstance(dst.loc, OclMemory): # host to device + params = [(dst.loc.symbol, dst.loc.queue)] + queue_sym = dst.loc.symbol + call = clEnqueueWriteBuffer(queue_sym.copy(), dst.name, True, 0, dst.type.size, src.name) + elif isinstance(src.loc, OclMemory): # device to host + params = [(src.loc.symbol, src.loc.queue)] + queue_sym = src.loc.symbol + call = clEnqueueReadBuffer(queue_sym.copy(), src.name, True, 0, src.type.size, dst.name) + else: + raise ValueError("Copy between non-ocl devices.") + + assert dst.type is not None, str(dst) + + if hasattr(dst, 'mem'): + params.append((SymbolRef(dst.name, dst.type), dst.mem)) + if hasattr(src, 'mem'): + params.append((SymbolRef(src.name, src.type), src.mem)) + + call._lift_params = node._lift_params + call.lift( + includes=[CppInclude("OpenCL/OpenCL.h")], + params=params + ) + + return call + + +class ArgZipper(NodeTransformer): + def visit_FunctionDecl(self, node): + params = [] + param_names = set() + args = [] + for pair in node.params: + if isinstance(pair, tuple): + sym, val = pair + if sym.name not in param_names: + params.append(sym) + param_names.add(sym.name) + args.append(val) + if sym.name == 'answer': + self.answer = val + else: + params.append(pair) + node.params = params + self.extra_args = args + + return node + + +class Memory(object): + pass + +class MainMemory(Memory): + def allocate(self, length, dtype, name): + ty = np.ctypeslib.ndpointer(dtype)() + mem = np.empty([length], dtype=dtype) + return mem, SymbolRef(name, ty) + + def __str__(self): + return "MainMemory" + +class OclMemory(Memory): + _next_cq_id = 0 + def __init__(self, queue): + self.queue = queue + self.symbol = SymbolRef("queue%d" % self._next_cq_id, queue) + self._next_cq_id += 1 + + def allocate(self, length, dtype, name): + mem = cl.clCreateBuffer(self.queue.context, length * dtype.itemsize) + mem.ptr_type = np.ctypeslib.ndpointer(dtype)() + mem.ptr_type._global = True + return mem, SymbolRef(name, mem) + + def __str__(self): + return "OclMemory<%s>" % self.queue.device + +class DotWriter(object): + def __init__(self): + self._next_id = 0 + + def write(self, node, name=""): + n = 99 - self._next_id + with open("graph.%02d.%s.dot" % (n,name), 'w') as f: + f.write(node.to_dot()) + self._next_id += 1 + +# --------------------------------------------------------------------------- +# Specializer code - translator + +class OpTranslator(LazySpecializedFunction): + def get_tuning_driver(self): + from ctree.tune import BruteForceTuningDriver as TuningDriver + from ctree.tune import MinimizeTime + from ctree.tune import IntegerParameter + from ctree.tune import BooleanArrayParameter + from ctree.tune import IntegerArrayParameter + + """ + from ctree.opentuner.driver import OpenTunerDriver as TuningDriver + from opentuner.search.objective import MinimizeTime + from opentuner.search.manipulator import ConfigurationManipulator + from opentuner.search.manipulator import IntegerParameter + from opentuner.search.manipulator import BooleanArrayParameter + from opentuner.search.manipulator import IntegerArrayParameter + """ + + nMemorySpaces = len(cl.clGetDeviceIDs()) + + params = [ + BooleanArrayParameter("parallelize", 7), + IntegerArrayParameter("locs", 7, 0, nMemorySpaces), + BooleanArrayParameter("distribute", 4), + BooleanArrayParameter("fusion", 7), + BooleanArrayParameter("reassociate", 4), + ] + + """ + manip = ConfigurationManipulator() + for param in params: + manip.add_parameter(param) + return TuningDriver(manipulator=manip, objective=MinimizeTime()) + """ + + return TuningDriver(params, MinimizeTime()) + + from ctree.tune import ConstantTuningDriver + return ConstantTuningDriver({ + 'locs': (0, 0, 1, 1, 0, 1, 1), + 'fusion': (True, True, True, True, True, True), + 'distribute': (True, True, True, True), + 'reassociate': (True, True, True, True), + 'parallelize': (True,) * 7 + }) + + def args_to_subconfig(self, args): + """ + Analyze arguments and return a 'subconfig', a hashable object + that classifies them. Arguments with identical subconfigs + might be processed by the same generated code. + """ + ptrs = tuple(np.ctypeslib.ndpointer(a.dtype) for a in args) + return { + 'ptrs': ptrs, + 'len': len(args[0]), + } + + def transform(self, py_ast, program_config): + """ + Convert the Python AST to a C AST according to the directions + given in program_config. + """ + arg_config, tuner_config = program_config + dot = DotWriter() + + # hack yo + ComputedVector._next_id = 0 + CopiedVector._next_id = 0 + + # set up OpenCL context and memory spaces + import pycl + context = pycl.clCreateContextFromType(pycl.CL_DEVICE_TYPE_ALL) + queues = [pycl.clCreateCommandQueue(context, dev) for dev in context.devices] + c_func = ElementwiseFunction(context, queues) + + memories = [MainMemory()] + [OclMemory(q) for q in queues] + main_memory = memories[0] + ptrs = arg_config['ptrs'] + dtype, length = ptrs[0]._dtype_, arg_config['len'] + + # pull stuff out of autotuner + distribute_directives = tuner_config['distribute'] + reassoc_directives = tuner_config['reassociate'] + locs = [memories[loc] for loc in tuner_config['locs']] + fusion_directives = tuner_config['fusion'] + parallelize_directives = tuner_config['parallelize'] + + dot.write(py_ast) + + # run basic conversions + proj = PyBasicConversions().visit(py_ast) + dot.write(proj) + + # run platform-independent transformations + proj = ApplyDistributiveProperty(distribute_directives).visit(proj) + dot.write(proj) + + proj = ApplyAssociativeProperty(reassoc_directives).visit(proj) + dot.write(proj) + + # set parameter types + proj = VectorFinder(ptrs, main_memory).visit(proj) + dot.write(proj) + + proj = LocationTagger(locs).visit(proj) + dot.write(proj) + + proj = InsertIntermediates(main_memory).visit(proj) + dot.write(proj) + + proj = CopyInserter(main_memory).visit(proj) + dot.write(proj) + + proj = DoFusion(fusion_directives).visit(proj) + dot.write(proj) + + proj = AllocateIntermediates(dtype, length).visit(proj) + dot.write(proj, "postintermed") + + py_op = proj.find(FunctionDecl, name="py_op") + schedules = FindParallelism(parallelize_directives).visit(py_op) + py_op.defn = parallelize_tasks(schedules) + dot.write(proj, "postparallel") + + proj = KernelOutliner(length).visit(proj) + dot.write(proj) + + proj = LowerKernelCalls().visit(proj) + dot.write(proj) + + proj = RefConverter().visit(proj) + dot.write(proj) + + proj = LowerLoopsAndCopies(length).visit(proj) + dot.write(proj) + + zipper = ArgZipper() + proj = zipper.visit( Lifter().visit(proj) ) + c_func.extra_args = zipper.extra_args + c_func.answer = zipper.answer + dot.write(proj) + + fn = proj.find(FunctionDecl) + return c_func.finalize("py_op", proj, fn.get_type()) + +class ElementwiseFunction(ConcreteSpecializedFunction): + def __init__(self, context, queues): + self.context = context + self.queues = queues + + def finalize(self, entry_name, proj, typesig): + self._c_function = self._compile(entry_name, proj, typesig) + return self + + def __call__(self, *args): + full_args = list(args) + self.extra_args + self._c_function(*full_args) + return np.copy(self.answer) + + +class Elementwise(object): + """ + A class for managing independent operation on elements + in numpy arrays. + """ + + def __init__(self, fn): + """Instantiate translator.""" + self.jit = OpTranslator(get_ast(fn)) + + def __call__(self, *args): + """Apply the operator to the arguments via a generated function.""" + return self.jit(*args) + + +# --------------------------------------------------------------------------- +# User code + +def py_op(a, b, c, d): + return (a + d) * (b + c) + +def main(): + n = 1234 + c_op = Elementwise(py_op) + + # doubling doubles + for i in range(2000): + a = np.arange(0*n, 1*n, dtype=np.float32()) + b = np.arange(1*n, 2*n, dtype=np.float32()) + c = np.arange(2*n, 3*n, dtype=np.float32()) + d = np.arange(3*n, 4*n, dtype=np.float32()) + + actual = c_op(a, b, c, d) + expected = py_op(a, b, c, d) + + np.testing.assert_array_equal(actual, expected) + + print("Success.") + + +if __name__ == '__main__': + main() diff --git a/examples/Fibonacci.py b/examples/Fibonacci.py index a3038bb..6e205a9 100644 --- a/examples/Fibonacci.py +++ b/examples/Fibonacci.py @@ -10,11 +10,11 @@ def fib(n): return fib(n-1) + fib(n-2) """ +from ctypes import * from ctree.c.nodes import * -from ctree.c.types import * fib_ast = \ - FunctionDecl(Int(), "fib", [SymbolRef("n", Int())], [ + FunctionDecl(c_int(), "fib", [SymbolRef("n", c_int())], [ If(Lt(SymbolRef("n"), Constant(2)), [Return(SymbolRef("n"))], [Return(Add(FunctionCall(SymbolRef("fib"), [Sub(SymbolRef("n"), Constant(1))]), diff --git a/examples/OclDoubler.py b/examples/OclDoubler.py index 4448bf4..08ced57 100644 --- a/examples/OclDoubler.py +++ b/examples/OclDoubler.py @@ -7,22 +7,43 @@ logging.basicConfig(level=20) import numpy as np +import ctypes as ct +import pycl as cl +import ctree.np from ctree.c.nodes import * -from ctree.c.types import * from ctree.cpp.nodes import * from ctree.ocl.nodes import * from ctree.ocl.types import * from ctree.ocl.macros import * -from ctree.templates.nodes import FileTemplate +from ctree.templates.nodes import StringTemplate from ctree.transformations import * +from ctree.frontend import get_ast from ctree.jit import LazySpecializedFunction -from ctree.types import get_ctree_type -from ctree.dotgen import to_dot +from ctree.jit import ConcreteSpecializedFunction + +from ctree import browser_show_ast # --------------------------------------------------------------------------- # Specializer code +class OpFunction(ConcreteSpecializedFunction): + def __init__(self): + self.context = cl.clCreateContextFromType() + self.queue = cl.clCreateCommandQueue(self.context) + + def finalize(self, kernel, tree, entry_name, entry_type): + self.kernel = kernel + self._c_function = self._compile(entry_name, tree, entry_type) + return self + + def __call__(self, A): + buf, evt = cl.buffer_from_ndarray(self.queue, A, blocking=False) + self._c_function(self.queue, self.kernel, buf) + B, evt = cl.buffer_to_ndarray(self.queue, buf, like=A) + return B + + class OpTranslator(LazySpecializedFunction): def args_to_subconfig(self, args): """ @@ -31,121 +52,98 @@ def args_to_subconfig(self, args): might be processed by the same generated code. """ A = args[0] - return len(A), A.dtype, A.ndim, A.shape + return np.ctypeslib.ndpointer(A.dtype, A.ndim, A.shape) def transform(self, py_ast, program_config): """ Convert the Python AST to a C AST according to the directions given in program_config. """ - len_A, A_dtype, A_ndim, A_shape = program_config[0] - A_type = NdPointer(A_dtype, A_ndim, A_shape) - + A = program_config[0] + len_A = np.prod(A._shape_) + inner_type = A._dtype_.type() + # browser_show_ast(py_ast,'tmp.png') apply_one = PyBasicConversions().visit(py_ast.body[0]) - apply_one_typesig = FuncType(A_type.get_base_type(), [A_type.get_base_type()]) - apply_one.set_typesig(apply_one_typesig) - - apply_kernel = FunctionDecl(Void(), "apply_kernel", - params=[SymbolRef("A", A_type)], - defn=[ - Assign(SymbolRef("i", Int()), - FunctionCall(SymbolRef("get_global_id"), [Constant(0)])), - If(Lt(SymbolRef("i"), Constant(len_A)), [ - Assign(ArrayRef(SymbolRef("A"), SymbolRef("i")), - FunctionCall(SymbolRef("apply"), - [ArrayRef(SymbolRef("A"), SymbolRef("i"))])) - ]) - ]) - - # add opencl type qualifiers - apply_kernel.set_kernel() - apply_kernel.params[0].set_global() + apply_one.name = 'apply' + apply_one.return_type = inner_type + apply_one.params[0].type = inner_type + + apply_kernel = FunctionDecl(None, "apply_kernel", + params=[SymbolRef("A", A()).set_global()], + defn=[ + Assign(SymbolRef("i", ct.c_int()), get_global_id(0)), + If(Lt(SymbolRef("i"), Constant(len_A)), [ + Assign(ArrayRef(SymbolRef("A"), SymbolRef("i")), + FunctionCall(SymbolRef(apply_one.name), + [ArrayRef(SymbolRef("A"), SymbolRef("i"))])), + ], []), + ] + ).set_kernel() kernel = OclFile("kernel", [apply_one, apply_kernel]) - template_args = { - 'array_decl': SymbolRef("data", A_type), - 'array_ref': SymbolRef("data"), - 'count': Constant(len_A), - 'kernel_path': kernel.get_generated_path_ref(), - 'kernel_name': String(apply_kernel.name), - } - template_path = os.path.join(os.getcwd(), "templates", "OclDoubler.tmpl.c") - - control = CFile("control", [ - FileTemplate(template_path, template_args), - ]) - tree = Project([kernel, control]) + control = StringTemplate(r""" + #ifdef __APPLE__ + #include + #else + #include + #endif + void apply_all(cl_command_queue queue, cl_kernel kernel, cl_mem buf) { + size_t global = $n; + size_t local = 32; + clSetKernelArg(kernel, 0, sizeof(cl_mem), &buf); + clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &global, &local, 0, NULL, NULL); - with open("graph.dot", 'w') as f: - f.write( to_dot(tree) ) - - entry_point_typesig = FuncType(Int(), [A_type]).as_ctype() - return tree, entry_point_typesig + } + """, {'n': Constant(len_A + 32 - (len_A % 32))}) + cfile = CFile("generated", [control], config_target='opencl') + return kernel, cfile + def finalize(self, transform_result, program_config): + kernel, cfile = transform_result + proj = Project([kernel, cfile]) + fn = OpFunction() -class ArrayOp(object): - """ - A class for managing independent operation on elements - in numpy arrays. - """ + program = cl.clCreateProgramWithSource(fn.context, kernel.codegen()).build() + apply_kernel_ptr = program['apply_kernel'] - def __init__(self): - """Instantiate translator.""" - from ctree.frontend import get_ast + entry_type = ct.CFUNCTYPE(None, cl.cl_command_queue, cl.cl_kernel, cl.cl_mem) + return fn.finalize(apply_kernel_ptr, proj, "apply_all", entry_type) - self.c_apply_all = OpTranslator(get_ast(self.apply), "apply_all") - - def __call__(self, A): - """Apply the operator to the arguments via a generated function.""" - retval = self.c_apply_all(A) - assert retval == 0, "Specialized function exited with non-zero value: %d" % retval + def interpret(self, A): + return np.vectorize(self.apply)(A) # --------------------------------------------------------------------------- -# User code +# user code -class Doubler(ArrayOp): - """Double elements of the array.""" - def apply(x): - return x * 2 +def double(x): + return x * 2 -class Squarer(ArrayOp): - """Double elements of the array.""" +def square(x): + return x * x - def apply(x): - return x * x +def main(): + doubler = OpTranslator.from_function(double, 'Doubler') + squarer = OpTranslator.from_function(square, 'Squarer') -def py_doubler(A): - for i in range(len(A)): - A[i] *= 2 - -def py_squarer(A): - for i in range(len(A)): - A[i] *= A[i] + data = np.arange(123, dtype=np.float32) -def main(): # squaring floats - c_squarer = Squarer() - actual_d = np.ones(1024, dtype=np.float32) - expected_d = np.ones(1024, dtype=np.float32) - c_squarer(actual_d) - py_squarer(expected_d) - np.testing.assert_array_equal(actual_d, expected_d) + actual = squarer(data) + expected = np.vectorize(square)(data) + np.testing.assert_array_equal(actual, expected) print("Squarer works.") # doubling floats - c_doubler = Doubler() - actual_d = np.ones(1024, dtype=np.float32) - expected_d = np.ones(1024, dtype=np.float32) - c_doubler(actual_d) - py_doubler(expected_d) - np.testing.assert_array_equal(actual_d, expected_d) + actual = doubler(data) + expected = np.vectorize(double)(data) + np.testing.assert_array_equal(actual, expected) print("Doubler works.") - if __name__ == '__main__': + # Testing conventional (non-lambda) kernel function implementation main() diff --git a/examples/OmpSpecializer.py b/examples/OmpSpecializer.py index 6528b39..0b62279 100644 --- a/examples/OmpSpecializer.py +++ b/examples/OmpSpecializer.py @@ -11,22 +11,31 @@ from ctree.nodes import Project from ctree.c.nodes import * from ctree.c.macros import * -from ctree.c.types import Void from ctree.cpp.nodes import * from ctree.omp.nodes import * from ctree.omp.macros import * -from ctree.jit import LazySpecializedFunction +from ctree.jit import LazySpecializedFunction, ConcreteSpecializedFunction +from ctypes import CFUNCTYPE # --------------------------------------------------------------------------- # Specializer code +class GreeterFunction(ConcreteSpecializedFunction): + + def finalize(self, tree, entry_name, entry_type): + self._c_function = self._compile(entry_name, tree, entry_type) + return self + + def __call__(self): + self._c_function() + class GreeterTranslator(LazySpecializedFunction): def transform(self, py_ast, program_config): tree = CFile("generated", [ CppInclude("omp.h"), CppInclude("stdio.h"), - FunctionDecl(Void(), "hello", + FunctionDecl(None, "hello", params=[], defn=[ OmpParallel( [OmpNumThreadsClause(Constant(4))] ), @@ -34,16 +43,25 @@ def transform(self, py_ast, program_config): omp_get_thread_num(), omp_get_num_threads()), ] ), - ]) - entry_point_typesig = tree.find(FunctionDecl, name="hello").get_type().as_ctype() + ], 'omp') + # entry_point_typesig = tree.find(FunctionDecl, name="hello").get_type().as_ctype() + + return tree + + def finalize(self, transform_result, program_config): + tree = transform_result[0] + entry_type = CFUNCTYPE(None) + + fn = GreeterFunction() - return Project([tree]), entry_point_typesig + return fn.finalize(Project([tree]), "hello", entry_type) class ParallelGreeter(object): def __init__(self): """Instantiate translator.""" - self.c_hello = GreeterTranslator(None, "hello") + import ast + self.c_hello = GreeterTranslator(ast.Module()) def __call__(self): """Apply the operator to the arguments via a generated function.""" diff --git a/examples/SimpleTranslator.py b/examples/SimpleTranslator.py index 525364e..116624b 100644 --- a/examples/SimpleTranslator.py +++ b/examples/SimpleTranslator.py @@ -7,12 +7,14 @@ logging.basicConfig(level=20) import numpy as np +import ctypes as ct -from ctree.c.types import FuncType from ctree.transformations import * -from ctree.frontend import get_ast +from ctree.frontend import get_ast, dump from ctree.jit import LazySpecializedFunction -from ctree.types import get_ctree_type +from ctree.jit import ConcreteSpecializedFunction +from ctree.types import get_ctype +from ctree.nodes import Project def fib(n): @@ -22,27 +24,49 @@ def fib(n): return fib(n - 1) + fib(n - 2) +class BasicFunction(ConcreteSpecializedFunction): + def __init__(self, entry_name, project_node, entry_typesig): + self._c_function = self._compile(entry_name, project_node, entry_typesig) + + + def __call__(self, *args, **kwargs): + return self._c_function(*args, **kwargs) + + class BasicTranslator(LazySpecializedFunction): - def __init__(self, func): - super(BasicTranslator, self).__init__(get_ast(func), func.__name__) def args_to_subconfig(self, args): - return {'arg_type': get_ctree_type(args[0])} + return {'arg_type': type(get_ctype(args[0]))} def transform(self, tree, program_config): """Convert the Python AST to a C AST.""" + tree = PyBasicConversions().visit(tree) - fib_fn = tree.find(FunctionDecl, name="fib") - fib_arg_type = program_config[0]['arg_type'] - fib_type = FuncType(fib_arg_type, [fib_arg_type]) - fib_fn.set_typesig(fib_type) + fib_fn = tree.find(FunctionDecl, name="apply") + arg_type = program_config[0]['arg_type'] + fib_fn.return_type = arg_type() + fib_fn.params[0].type = arg_type() + c_translator = CFile("generated", [tree]) - return tree, fib_type.as_ctype() + return [c_translator] + + def finalize(self, transform_result, program_config): + + c_translator = transform_result[0] + proj = Project([c_translator]) + + arg_config, tuner_config = program_config + arg_type = arg_config['arg_type'] + entry_type = ct.CFUNCTYPE(arg_type, arg_type) + + return BasicFunction("apply", proj, entry_type) def main(): - c_fib = BasicTranslator(fib) + + # create a class called Doubler that has the function double(n) as an @staticmethod + c_fib = BasicTranslator.from_function(fib, "Translator") assert fib(10) == c_fib(10) assert fib(11) == c_fib(11) diff --git a/examples/TemplateDoubler.py b/examples/TemplateDoubler.py index 772f41d..f5bd280 100644 --- a/examples/TemplateDoubler.py +++ b/examples/TemplateDoubler.py @@ -8,14 +8,15 @@ import numpy as np +import ctree.np +from ctypes import * from ctree.frontend import get_ast from ctree.c.nodes import * -from ctree.c.types import * from ctree.templates.nodes import * -from ctree.dotgen import to_dot from ctree.transformations import * from ctree.jit import LazySpecializedFunction -from ctree.types import get_ctree_type +from ctree.jit import ConcreteSpecializedFunction +from ctree.nodes import Project # --------------------------------------------------------------------------- # Specializer code @@ -29,12 +30,7 @@ def args_to_subconfig(self, args): might be processed by the same generated code. """ A = args[0] - return { - 'A_len': len(A), - 'A_dtype': A.dtype, - 'A_ndim': A.ndim, - 'A_shape': A.shape, - } + return {'ptr': np.ctypeslib.ndpointer(A.dtype, A.ndim, A.shape)} def transform(self, py_ast, program_config): """ @@ -42,19 +38,14 @@ def transform(self, py_ast, program_config): given in program_config. """ arg_config, tuner_config = program_config - len_A = arg_config['A_len'] - A_dtype = arg_config['A_dtype'] - A_ndim = arg_config['A_ndim'] - A_shape = arg_config['A_shape'] - - inner_type = get_ctree_type(A_dtype) - array_type = NdPointer(A_dtype, A_ndim, A_shape) - apply_one_typesig = FuncType(inner_type, [inner_type]) + A = arg_config['ptr'] + inner_type = A._dtype_.type() + nItems = np.prod(A._shape_) template_entries = { - 'array_decl': SymbolRef("A", array_type), + 'array_decl': SymbolRef("A", A()), 'array_ref' : SymbolRef("A"), - 'num_items' : Constant(len_A), + 'num_items' : Constant(nItems), } tree = CFile("generated", [ @@ -69,49 +60,45 @@ def transform(self, py_ast, program_config): ]) tree = PyBasicConversions().visit(tree) + print(tree) apply_one = tree.find(FunctionDecl, name="apply") apply_one.set_static().set_inline() - apply_one.set_typesig(apply_one_typesig) + apply_one.return_type = inner_type + apply_one.params[0].type = inner_type + return (tree,) - with open("graph.dot", 'w') as f: - f.write( to_dot(tree) ) + def finalize(self, transform_result, program_config): + tree = transform_result[0] + proj = Project([tree]) + arg_config = program_config[0] + A = arg_config['ptr'] + entry_point_typesig = CFUNCTYPE(None, A) - entry_point_typesig = FuncType(Void(), [array_type]).as_ctype() - return Project([tree]), entry_point_typesig + return BasicFunction("apply_all", proj, entry_point_typesig) -class ArrayOp(object): - """ - A class for managing independent operation on elements - in numpy arrays. - """ +class BasicFunction(ConcreteSpecializedFunction): + def __init__(self, entry_name, proj_node, entry_typesig): + self._c_function = self._compile(entry_name, proj_node, entry_typesig) - def __init__(self): - """Instantiate translator.""" - self.c_apply_all = OpTranslator(get_ast(self.apply), "apply_all") - - def __call__(self, A): - """Apply the operator to the arguments via a generated function.""" - return self.c_apply_all(A) + def __call__(self, *args, **kwargs): + return self._c_function(*args, **kwargs) # --------------------------------------------------------------------------- # User code -class Doubler(ArrayOp): - """Double elements of the array.""" - - def apply(n): - return n * 2 +def double(n): + return n * 2 def py_doubler(A): A *= 2 def main(): - c_doubler = Doubler() + c_doubler = OpTranslator.from_function(double) # doubling doubles actual_d = np.ones(12, dtype=np.float64) diff --git a/examples/__init__.py b/examples/__init__.py index 642391e..ee9d84c 100644 --- a/examples/__init__.py +++ b/examples/__init__.py @@ -1 +1,16 @@ __author__ = 'Chick Markley' + +__all__ = [ + "ArrayDoubler", + "AstToDot", + "dgemm", + "Distrib", + "Fibonacci", + "hwacha", + "OclDoubler", + "OmpSpecializer", + "SimpleTranslator", + "stencil_grid/", + "TemplateDoubler", + "TuningSpecializer", +] \ No newline at end of file diff --git a/examples/dgemm.py b/examples/dgemm.py index 9c53678..2c99d45 100644 --- a/examples/dgemm.py +++ b/examples/dgemm.py @@ -1,24 +1,34 @@ """ Computes matrix-matrix products via specialization. + +The C configuration inside ctree.cfg should include the -mavx flag in the cflags section. +For example: +[c] +cc = gcc-4.9 +cflags = -mavx -O3 -mmacosx-version-min=10.6 -std=c99 + +This program also requires the current ucb-sejits fork of opentuner: +https://github.com/ucb-sejits/opentuner """ import logging +from ctree.nodes import Project logging.basicConfig(level=60) import copy import numpy as np +import ctypes as ct +import inspect from ctree.c.nodes import * -from ctree.cpp.nodes import Comment -from ctree.c.types import * +from ctree.cpp.nodes import CppComment +from ctree.types import * from ctree.simd.macros import * from ctree.simd.types import * from ctree.templates.nodes import StringTemplate -from ctree.dotgen import to_dot from ctree.transformations import * -from ctree.jit import LazySpecializedFunction -from ctree.types import get_ctree_type +from ctree.jit import LazySpecializedFunction, ConcreteSpecializedFunction from ctree.metrics.watts_up_reader import WattsUpReader def MultiArrayRef(name, *idxs): @@ -26,7 +36,7 @@ def MultiArrayRef(name, *idxs): Given a string and a list of ints, produce the chain of array-ref expressions: - >>> MultiArrayRef('foo', 1, 2, 3).codegen() + >>> MultiArrayRef('foo', Constant(1), Constant(2), Constant(3)).codegen() 'foo[1][2][3]' """ tree = ArrayRef(SymbolRef(name), idxs[0]) @@ -34,10 +44,14 @@ def MultiArrayRef(name, *idxs): tree = ArrayRef(tree, Constant(idx)) return tree +def dummy_func(): + return + + class DgemmTranslator(LazySpecializedFunction): def __init__(self): self._current_config = None - super(DgemmTranslator, self).__init__(None, "dgemm") + super(DgemmTranslator, self).__init__(ast.parse(inspect.getsource(dummy_func)), "dgemm") def get_tuning_driver(self): from ctree.opentuner.driver import OpenTunerDriver @@ -52,7 +66,7 @@ def get_tuning_driver(self): manip.add_parameter(IntegerParameter("cx", 8, 32)) manip.add_parameter(IntegerParameter("cy", 8, 32)) - return OpenTunerDriver(manipulator=manip, objective=MinimizeEnergy()) + return OpenTunerDriver(manipulator=manip, objective=MinimizeTime()) def args_to_subconfig(self, args): """ @@ -62,7 +76,7 @@ def args_to_subconfig(self, args): """ C, A, B, duration = args n = len(A) - assert C.shape == A.shape == B.shape == (n,n) + assert C.shape == A.shape == B.shape == (n, n) assert A.dtype == B.dtype == C.dtype return { 'n': n, @@ -73,10 +87,10 @@ def _gen_load_c_block(self, rx, ry, lda): """ Return a subtree that loads a block of 'c'. """ - stmts = [Comment("Load a block of c")] + stmts = [CppComment("Load a block of c")] for j in range(rx): for i in range(ry/4): - stmt = Assign(MultiArrayRef("c", i, j), + stmt = Assign(MultiArrayRef("c", Constant(i), Constant(j)), mm256_loadu_pd(Add(SymbolRef("C"), Constant(i*4+j*lda)))) stmts.append(stmt) return Block(stmts) @@ -85,11 +99,11 @@ def _gen_store_c_block(self, rx, ry, lda): """ Return a subtree that loads a block of 'c'. """ - stmts = [Comment("Store the c block")] + stmts = [CppComment("Store the c block")] for j in range(rx): for i in range(ry/4): stmt = mm256_storeu_pd(Add(SymbolRef("C"), Constant(i*4+j*lda)), - MultiArrayRef("c", i, j)) + MultiArrayRef("c", Constant(i), Constant(j))) stmts.append(stmt) return Block(stmts) @@ -108,14 +122,14 @@ def _gen_rank1_update(self, i, rx, ry, cx, cy, lda): stmts.append(stmt) for k in range(ry/4): - stmt = Assign(MultiArrayRef("c", k, j), - mm256_add_pd( MultiArrayRef("c", k, j), + stmt = Assign(MultiArrayRef("c", Constant(k), Constant(j)), + mm256_add_pd( MultiArrayRef("c", Constant(k), Constant(j)), mm256_mul_pd(SymbolRef("a%d"%k), SymbolRef("b")) )) stmts.append(stmt) return Block(stmts) def _gen_k_rank1_updates(self, rx, ry, cx, cy, unroll, lda): - stmts = [Comment("do K rank-1 updates")] + stmts = [CppComment("do K rank-1 updates")] for i in range(ry/4): stmts.append(SymbolRef("a%d" % i, m256d())) stmts.append(SymbolRef("b", m256d())) @@ -130,19 +144,16 @@ def transform(self, py_ast, program_config): self._current_config = program_config arg_config, tuner_config = program_config - n, dtype = arg_config['n'], arg_config['dtype'] + n, dtype = arg_config['n'], arg_config['dtype'] rx, ry = tuner_config['rx']*4, tuner_config['ry']*4 cx, cy = tuner_config['cx']*4, tuner_config['cy']*4 unroll = tuner_config['ry']*4 - elem_type = get_ctree_type(dtype) - array_type = NdPointer(dtype, 2, (n,n)) - - dgemm_typesig = FuncType(Void(), [array_type, array_type, array_type, Ptr(Double())]) + array_type = np.ctypeslib.ndpointer(dtype, 2, (n, n)) - A = SymbolRef("A", array_type) - B = SymbolRef("B", array_type) - C = SymbolRef("C", array_type) + A = SymbolRef("A", array_type()) + B = SymbolRef("B", array_type()) + C = SymbolRef("C", array_type()) N = Constant(n) RX, RY = Constant(rx), Constant(ry) @@ -161,8 +172,9 @@ def transform(self, py_ast, program_config): "lda": N, } - preamble = StringTemplate(""" + preamble = StringTemplate(""" #include + #include #define min(x,y) (((x)<(y))?(x):(y)) """, copy.deepcopy(template_args)) @@ -193,7 +205,7 @@ def transform(self, py_ast, program_config): fast_dgemm = StringTemplate(""" void fast_dgemm( int M, int N, int K, $A_decl, $B_decl, $C_decl ) { - static double a[$CX*$CY] __attribute__ ((aligned (16))); + static double a[$CX*$CY] __attribute__ ((aligned (32))); // make a local aligned copy of A's block for( int j = 0; j < K; j++ ) @@ -227,7 +239,7 @@ def transform(self, py_ast, program_config): """, {}) - dgemm = StringTemplate(""" + dgemm = StringTemplate(""" int align( int x, int y ) { return x <= y ? x : (x/y)*y; } void dgemm($C_decl, $A_decl, $B_decl, double *duration) { @@ -264,7 +276,30 @@ def transform(self, py_ast, program_config): dgemm, ]) - return Project([tree]), dgemm_typesig.as_ctype() + c_dgemm = CFile("generated", [tree]) + return [c_dgemm] + + def finalize(self, transform_result, program_config): + c_dgemm = transform_result[0] + proj = Project([c_dgemm]) + + arg_config, tuner_config = program_config + n, dtype = arg_config['n'], arg_config['dtype'] + array_type = np.ctypeslib.ndpointer(dtype, 2, (n, n)) + entry_type = ct.CFUNCTYPE(None, array_type, array_type, array_type, POINTER(c_double)) + + + concrete_Fn = ConcreteDgemm() + return concrete_Fn.finalize("dgemm", proj, entry_type) + + +class ConcreteDgemm(ConcreteSpecializedFunction): + def finalize(self, entry_point_name, project_node, entry_typesig): + self._c_function = self._compile(entry_point_name, project_node, entry_typesig) + return self + + def __call__(self, C, A, B, duration): + return self._c_function(C, A, B, duration) class SquareDgemm(object): @@ -274,19 +309,38 @@ def __init__(self): def __call__(self, A, B): """C = A * B""" - from ctypes import c_double, byref C = np.zeros(shape=A.shape, dtype=A.dtype) - duration = c_double() - meter = WattsUpReader() + meter = Meter() meter.start_recording() - self.c_dgemm(C, A, B, byref(duration)) - joules = meter.get_recording()[0].joules - seconds = duration.value + self.c_dgemm(C, A, B, ct.byref(meter.time_meter)) + + joules = meter.energy_value() + seconds = meter.time_value() self.c_dgemm.report(time=seconds, energy=joules) return C, seconds, joules, self.c_dgemm._current_config +class Meter(object): + def __init__(self, use_energy=False): + self.time_meter = c_double() + self.use_energy = use_energy + self.energy_meter = WattsUpReader() if self.use_energy else None + + def start_recording(self): + if self.use_energy: + self.energy_meter.start_recording() + + def time_value(self): + return self.time_meter.value + + def energy_value(self): + if self.use_energy: + return self.energy_meter.get_recording()[0].joules + else: + return 0.0 + + def main(): n = 2048 c_dot = SquareDgemm() @@ -297,17 +351,18 @@ def main(): best_joules = float('inf') for i in range(1000): - C_actual, seconds, joules, config = c_dot(A, B) - np.testing.assert_almost_equal(C_actual.T, C_expected) + C_actual, seconds, joules, config = c_dot(A, B) + np.testing.assert_almost_equal(C_actual.T, C_expected) - best_indicator = "*** new best ***" if joules < best_joules else "" - best_joules = min(best_joules, joules) + best_indicator = "*** new best ***" if joules < best_joules else "" + best_joules = min(best_joules, joules) - ticks = min(40, int(joules / 10.0)) - print ("trial %s %s took %f sec, used %s joules: %s %s" % \ - (str(i).rjust(3), str(config[1]).ljust(38), seconds, str(joules).rjust(5), ('#' * ticks).ljust(40), best_indicator)) + ticks = min(40, int(joules / 10.0)) + print("trial %s %s took %f sec, used %s joules: %s %s" % + (str(i).rjust(3), str(config[1]).ljust(38), seconds, str(joules).rjust(5), + ('#' * ticks).ljust(40), best_indicator)) - del C_actual + del C_actual print("Done.") diff --git a/examples/hwacha.py b/examples/hwacha.py new file mode 100644 index 0000000..f59ef80 --- /dev/null +++ b/examples/hwacha.py @@ -0,0 +1,429 @@ +import numpy as np +import ctypes as ct +import ast + +from ctree.jit import LazySpecializedFunction, ConcreteSpecializedFunction +from ctree.transformations import PyBasicConversions +from ctree.transforms.declaration_filler import DeclarationFiller +from ctree.c.nodes import CFile +import ctree.c.nodes as C +from ctree.nodes import Project +from ctree.types import get_ctype +from ctree.templates.nodes import StringTemplate + + +def get_nd_pointer(arg): + return np.ctypeslib.ndpointer(arg.dtype, arg.ndim, arg.shape) + + +class HwachaFN(ConcreteSpecializedFunction): + def finalize(self, entry_point_name, project_node, entry_typesig): + self._c_function = self._compile(entry_point_name, project_node, + entry_typesig) + return self + + def __call__(self, *args): + return self._c_function(*args) + + +class MapTransformer(ast.NodeTransformer): + def __init__(self, loopvar, param_dict, retval_name): + self.loopvar = loopvar + self.param_dict = param_dict + self.retval_name = retval_name + + def visit_SymbolRef(self, node): + if node.name in self.param_dict: + return C.ArrayRef(node, C.SymbolRef(self.loopvar)) + return node + + def visit_Return(self, node): + node.value = self.visit(node.value) + return C.Assign(C.ArrayRef(C.SymbolRef(self.retval_name), + C.SymbolRef(self.loopvar)), + node.value) + + +hwacha_configure_block = """ +size_t vector_length; +__asm__ volatile ( + "vsetcfg 16, 1\\n" + "vsetvl %0, %1\\n" + : "=r"(vector_length) + : "r"({SIZE}) +); +""" + +bounds_check = """ +if ({SIZE} == {loopvar}) continue; +""" + +class ScalarFinder(ast.NodeVisitor): + def __init__(self, scalars): + self.scalars = scalars + + def visit_Constant(self, node): + self.scalars.add(node.value) + +def get_scalars_in_body(node): + scalars = set() + visitor = ScalarFinder(scalars) + for stmt in node.body: + visitor.visit(stmt) + return scalars + +number_dict = { + "1": "one", + "2": "two", + "3": "three", + "4": "four", + "5": "five", + "6": "six", + "7": "seven", + "8": "eight", + "9": "nine", + "0": "zero", + ".": "dot" +} + +def scalar_init(scalar): + name = "".join(number_dict[digit] for digit in str(scalar)) + return StringTemplate(""" +union {{ + float f; + uint32_t i; +}} {name}; +{name}.f = {scalar}f; + """.format(name=name, scalar=scalar)) + +obtained_vector_length = """ +size_t obtained_vector_length; +__asm__ volatile( + "vsetvl %0, %1\\n" + : "=r"(obtained_vector_length) + : "r"({SIZE} - {loopvar}) + ); +assert(obtained_vector_length <= {SIZE}); +""" + +class ArrayRefFinder(ast.NodeVisitor): + def __init__(self, refs): + self.refs = refs + + def visit_BinaryOp(self, node): + if isinstance(node.op, C.Op.ArrayRef): + self.refs.append(node) + else: + self.visit(node.left) + self.visit(node.right) + +def get_array_references_in_body(node): + refs = [] + finder = ArrayRefFinder(refs) + for stmt in node.body: + finder.visit(stmt) + return refs + +class HwachaASMTranslator(ast.NodeTransformer): + def __init__(self, scalars, ref_register_map, body, type_map): + self.scalars = scalars + self.ref_register_map = ref_register_map + self.body = body + self.curr_register = -1 + self.reg_map = {} + self.type_map = type_map + + def get_next_register(self): + self.curr_register += 1 + return "vv{}".format(self.curr_register) + + def visit_SymbolRef(self, node): + if node.name in self.reg_map: + return self.reg_map[node.name] + return node + + def visit_Cast(self, node): + reg = self.get_next_register() + value = self.visit(node.value) + if isinstance(node.type, ct.c_float): + self.body.append(" vfcvt.s.w {0}, {1}\\n".format(reg, value)) + self.type_map[reg] = ct.c_float + return reg + else: + raise NotImplementedError() + + def visit_Constant(self, node): + self.type_map[node.value] = get_ctype(node.value) + return self.scalars[node.value] + + def visit_FunctionCall(self, node): + if node.func.name == 'max': + arg1 = self.visit(node.args[0]) + arg2 = self.visit(node.args[1]) + reg = self.get_next_register() + print(node) + print(arg1) + if self.type_map[arg1] == ct.c_float or \ + self.type_map[arg2] == ct.c_float: + self.body.append(" vfmax.s {0}, {1}, {2}\\n".format( + reg, arg1, arg2 + )) + self.type_map[reg] = ct.c_float + return reg + elif node.func.name == 'min': + arg1 = self.visit(node.args[0]) + arg2 = self.visit(node.args[1]) + reg = self.get_next_register() + if self.type_map[arg1] == ct.c_float or \ + self.type_map[arg2] == ct.c_float: + self.body.append(" vfmin.s {0}, {1}, {2}\\n".format( + reg, arg1, arg2 + )) + self.type_map[reg] = ct.c_float + return reg + raise NotImplementedError() + + def visit_BinaryOp(self, node): + if isinstance(node.op, C.Op.ArrayRef): + reg = self.get_next_register() + self.body.append(" vlwu {0}, {1}\\n".format( + reg, + self.ref_register_map[str(node)][1])) + return reg + if isinstance(node.op, C.Op.Assign): + node.right = self.visit(node.right) + if isinstance(node.left, C.SymbolRef): + self.reg_map[node.left.name] = node.right + return + elif isinstance(node.left, C.BinaryOp) and \ + isinstance(node.left.op, C.Op.ArrayRef): + if self.type_map[node.left.left.name] != self.type_map[node.right]: + reg = self.get_next_register() + self.body.append(" vfcvt.w.s {0}, {1}\\n".format(reg, node.right)) + self.body.append(" vsw {0}, {1}\\n".format(reg, + self.ref_register_map[str(node.left)][1])) + return + + node.left = self.visit(node.left) + node.right = self.visit(node.right) + reg = self.get_next_register() + if isinstance(node.op, C.Op.Sub): + self.body.append(" vsub {0}, {1}, {2}\\n".format( + reg, node.left, node.right)) + elif isinstance(node.op, C.Op.Div): + if self.type_map[node.left] == ct.c_float or \ + self.type_map[node.right] == ct.c_float: + self.body.append(" vfdiv.s {0}, {1}, {2}\\n".format( + reg, node.left, node.right)) + self.type_map[reg] = ct.c_float + else: + raise NotImplementedError() + elif isinstance(node.op, C.Op.Mul): + if self.type_map[node.left] == ct.c_float or \ + self.type_map[node.right] == ct.c_float: + self.body.append(" vfmul.s {0}, {1}, {2}\\n".format( + reg, node.left, node.right)) + self.type_map[reg] = ct.c_float + else: + raise NotImplementedError() + return reg + +def get_asm_body(node, scalars, refs, type_map): + body = """ +__asm__ volatile ( +".align 3\\n" +"__hwacha_body:\\n" + """ + asm_body = [] + translator = HwachaASMTranslator(scalars, refs, asm_body, type_map) + for s in node.body: + translator.visit(s) + for s in asm_body: + body += "\"" + s + "\"\n" + body += "\" vstop\\n\"\n" + body += " );" + return StringTemplate(body) + + +class HwachaVectorize(ast.NodeTransformer): + def __init__(self, type_map, defns): + self.type_map = type_map + self.defns = defns + + def visit_For(self, node): + if node.pragma == "ivdep": + block = [] + loopvar = node.incr.arg + size = node.test.right + scalars = get_scalars_in_body(node) + refs = get_array_references_in_body(node) + ref_register_map = {} + scalar_register_map = {} + for index, ref in enumerate(refs): + ref_register_map[str(ref)] = (ref, "va{}".format(index)) + for index, scalar in enumerate(scalars): + reg = "vs{}".format(index) + scalar_register_map[scalar] = reg + self.type_map[reg] = get_ctype(scalar) + body = [] + block.append(StringTemplate(hwacha_configure_block.format(SIZE=size))) + + node.incr = C.AddAssign(loopvar, C.SymbolRef("vector_length")) + self.defns.append(get_asm_body(node, scalar_register_map, + ref_register_map, self.type_map)) + block.append(node) + + body.append(StringTemplate(bounds_check.format(SIZE=size, + loopvar=loopvar))) + + for scalar in scalars: + body.append(scalar_init(scalar)) + + body.append(StringTemplate(obtained_vector_length.format(SIZE=size, + loopvar=loopvar))) + + block1 = "" + block2 = "" + index = 0 + for _, info in ref_register_map.items(): + ref, register = info + block1 += "\t \"vmsa {0}, %{1}\\n\"\n".format(register, index) + block2 += "\"r\"({0} + {1}),\n".format( + ref.left.name, ref.right.name) + index += 1 + for scalar, register in scalar_register_map.items(): + block1 += "\t \"vmss {0}, %{1}\\n\"\n".format(register, index) + block2 += "\"r\"({0}.i),\n".format( + "".join(number_dict[digit] for digit in str(scalar))) + index += 1 + block1 += "\"fence\\n\"\n" + block1 += "\"vf 0(%{0})\\n\"\n".format(index) + block2 += "\"r\" (&__hwacha_body)" + body.append(StringTemplate( + """ +__asm__ volatile( +{block1} + : + : {block2} + : "memory" +); + """.format(block1=block1, block2=block2))) + + node.body = body + block.append( +StringTemplate(""" +__asm__ volatile( + "fence\\n" +); +""")) + return block + + + +class HwachaTranslator(LazySpecializedFunction): + def args_to_subconfig(self, args): + return tuple(get_nd_pointer(arg) for arg in args) + + def transform(self, py_ast, program_cfg): + arg_cfg, tune_cfg = program_cfg + tree = PyBasicConversions().visit(py_ast) + param_dict = {} + tree.body[0].params.append(C.SymbolRef("retval", arg_cfg[0]())) + # Annotate arguments + for param, type in zip(tree.body[0].params, arg_cfg): + param.type = type() + param_dict[param.name] = type._dtype_ + + length = np.prod(arg_cfg[0]._shape_) + transformer = MapTransformer("i", param_dict, "retval") + body = list(map(transformer.visit, tree.body[0].defn)) + + tree.body[0].defn = [C.For( + C.Assign(C.SymbolRef("i", ct.c_int()), C.Constant(0)), + C.Lt(C.SymbolRef("i"), C.Constant(length)), + C.PostInc(C.SymbolRef("i")), + body=body, + pragma="ivdep" + )] + + tree = DeclarationFiller().visit(tree) + defns = [] + tree = HwachaVectorize(param_dict, defns).visit(tree) + file_body = [ + StringTemplate("#include "), + StringTemplate("#include "), + StringTemplate("#include "), + StringTemplate("extern \"C\" void __hwacha_body(void);"), + ] + file_body.extend(defns) + file_body.append(tree) + return [CFile("generated", file_body)] + + def finalize(self, transform_result, program_config): + generated = transform_result[0] + print(generated) + proj = Project([generated]) + entry_type = ct.CFUNCTYPE(None, *program_config[0]) + return HwachaFN().finalize("apply", proj, entry_type) + + +def hwacha_map(fn, *args): + mapfn = HwachaTranslator.from_function(fn, "map") + retval = np.empty_like(args[0]) + args += (retval, ) + mapfn(*args) + return retval + + +CALIBRATE_COLD = 0x7000 +CALIBRATE_HOT = 0xA000 + +SIZE = (208 * 156) + +# Generate a dummy calibration table, just so there's something +# to execute. +cold = np.full(SIZE, CALIBRATE_COLD, np.int32) +hot = np.full(SIZE, CALIBRATE_HOT, np.int32) + +# Generate a dummy input image, again just so there's something +# to execute. +raw = np.empty(SIZE, np.int32) + +for i in range(SIZE): + scale = (CALIBRATE_HOT - CALIBRATE_COLD) + percent = (i % 120) - 10 + raw[i] = scale * (percent / 100.0) + CALIBRATE_COLD + raw[i] = CALIBRATE_COLD + (i % (int)(scale - 2)) + 1 + +def gold(cold, hot, raw, flat): + for i in range(208 * 156): + _max = hot[i] + _min = cold[i] + offset = raw[i] - _min + scale = _max - _min + foffset = float(offset) + fscale = float(scale) + scaled = foffset / fscale + scaled = min(1.0, scaled) + scaled = max(0.0, scaled) + flat[i] = 255 * scaled + +def test_map(cold, hot, raw): + _max = hot + _min = cold + offset = raw - _min + scale = _max - _min + foffset = float(offset) + fscale = float(scale) + scaled = foffset / fscale + scaled = min(1.0, scaled) + scaled = max(0.0, scaled) + return 255.0 * scaled + + +flat_gold = np.empty_like(raw) +gold(cold, hot, raw, flat_gold) + +flat_test = hwacha_map(test_map, cold, hot, raw) + +np.testing.assert_array_equal(flat_gold, flat_test) diff --git a/examples/stencil_grid/bilateral_filter.py b/examples/stencil_grid/bilateral_filter.py index 4d29ea5..0075a21 100644 --- a/examples/stencil_grid/bilateral_filter.py +++ b/examples/stencil_grid/bilateral_filter.py @@ -1,10 +1,9 @@ from examples.stencil_grid.stencil_kernel import * from examples.stencil_grid.stencil_grid import StencilGrid - +from ctree.util import Timer import sys import numpy import math -import time width = int(sys.argv[2]) height = int(sys.argv[3]) @@ -56,15 +55,6 @@ def distance(x, y): gaussian2 = gaussian(stdev_s, 256) -class Timer: - def __enter__(self): - self.start = time.clock() - return self - - def __exit__(self, *args): - self.end = time.clock() - self.interval = self.end - self.start - kernel.kernel(in_grid, gaussian1, gaussian2, out_grid) diff --git a/examples/stencil_grid/stencil_kernel.py b/examples/stencil_grid/stencil_kernel.py index 472762d..2b23102 100644 --- a/examples/stencil_grid/stencil_kernel.py +++ b/examples/stencil_grid/stencil_kernel.py @@ -51,7 +51,7 @@ def __init__(self, func, entry_point, input_grids, output_grid, constants): self.input_grids = input_grids self.output_grid = output_grid self.constants = constants - super(StencilConvert, self).__init__(get_ast(func), entry_point) + super(StencilConvert, self).__init__(get_ast(func)) def args_to_subconfig(self, args): conf = () diff --git a/examples/templates/OclDoubler.tmpl.c b/examples/templates/OclDoubler.tmpl.c deleted file mode 100644 index 5d9a941..0000000 --- a/examples/templates/OclDoubler.tmpl.c +++ /dev/null @@ -1,186 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -int apply_all($array_decl) -{ - const unsigned int count = $count; // number of elements in array - int err; // error code returned from api calls - - size_t global = $count; // global domain size for our calculation - size_t local = 32; // local domain size for our calculation - - cl_device_id device_id; // compute device id - cl_context context; // compute context - cl_command_queue commands; // compute command queue - cl_program program; // compute program - cl_kernel kernel; // compute kernel - - cl_mem device_data; // device memory used for the data array - - // Connect to a compute device - // - int gpu = 1; - err = clGetDeviceIDs(NULL, gpu ? CL_DEVICE_TYPE_GPU : CL_DEVICE_TYPE_CPU, 1, &device_id, NULL); - if (err != CL_SUCCESS) - { - printf("Error: Failed to create a device group!\n"); - return err; - } - - // Create a compute context - // - context = clCreateContext(0, 1, &device_id, NULL, NULL, &err); - if (!context) - { - printf("Error: Failed to create a compute context!\n"); - return err; - } - - // Create a command commands - // - commands = clCreateCommandQueue(context, device_id, 0, &err); - if (!commands) - { - printf("Error: Failed to create a command commands!\n"); - return err; - } - - // Read the kernel into a string - // - FILE *kernelFile = fopen($kernel_path, "rb"); - if (kernelFile == NULL) { - printf("Error: Coudn't open kernel file.\n"); - return err; - } - - fseek(kernelFile, 0 , SEEK_END); - long kernelFileSize = ftell(kernelFile); - rewind(kernelFile); - - // Allocate memory to hold kernel - // - char *KernelSource = malloc(kernelFileSize*sizeof(char)); - memset(KernelSource, 0, kernelFileSize); - if (KernelSource == NULL) { - printf("Error: failed to allocate memory to hold kernel text.\n"); - return err; - } - - // Read the kernel into memory - // - int result = fread(KernelSource, sizeof(char), kernelFileSize, kernelFile); - if (result != kernelFileSize) { - printf("Error: read fewer bytes of kernel text than expected.\n"); - return err; - } - fclose(kernelFile); - - // Create the compute program from the source buffer - // - program = clCreateProgramWithSource(context, 1, (const char **) & KernelSource, NULL, &err); - if (!program) - { - printf("Error: Failed to create compute program!\n"); - return err; - } - - // Build the program executable - // - err = clBuildProgram(program, 0, NULL, NULL, NULL, NULL); - if (err != CL_SUCCESS) - { - size_t len; - char buffer[2048]; - - printf("Error: Failed to build program executable!\n"); - clGetProgramBuildInfo(program, device_id, CL_PROGRAM_BUILD_LOG, sizeof(buffer), buffer, &len); - printf("%s\n", buffer); - return err; - } - - // Create the compute kernel in the program we wish to run - // - kernel = clCreateKernel(program, $kernel_name, &err); - if (!kernel || err != CL_SUCCESS) - { - printf("Error: Failed to create compute kernel!\n"); - return err; - } - - // Create the data array in device memory for our calculation - // - device_data = clCreateBuffer(context, CL_MEM_READ_WRITE, sizeof($array_ref[0]) * count, NULL, NULL); - if (!device_data) - { - printf("Error: Failed to allocate device memory!\n"); - return err; - } - - // Write our data set into the data array in device memory - // - err = clEnqueueWriteBuffer(commands, device_data, CL_TRUE, 0, sizeof($array_ref[0]) * count, $array_ref, 0, NULL, NULL); - if (err != CL_SUCCESS) - { - printf("Error: Failed to write to source array!\n"); - return err; - } - - // Set the arguments to our compute kernel - // - err = 0; - err = clSetKernelArg(kernel, 0, sizeof(cl_mem), &device_data); - if (err != CL_SUCCESS) - { - printf("Error: Failed to set kernel arguments! %d\n", err); - return err; - } - - // Get the maximum work group size for executing the kernel on the device - // - err = clGetKernelWorkGroupInfo(kernel, device_id, CL_KERNEL_WORK_GROUP_SIZE, sizeof(local), &local, NULL); - if (err != CL_SUCCESS) - { - printf("Error: Failed to retrieve kernel work group info! %d\n", err); - return err; - } - - // Execute the kernel over the entire range of our 1d input data set - // using the maximum number of work group items for this device - // - err = clEnqueueNDRangeKernel(commands, kernel, 1, NULL, &global, &local, 0, NULL, NULL); - if (err) - { - printf("Error: Failed to execute kernel!\n"); - return err; - } - - // Wait for the command commands to get serviced before reading back results - // - clFinish(commands); - - // Read back the results from the device to verify the output - // - err = clEnqueueReadBuffer( commands, device_data, CL_TRUE, 0, sizeof($array_ref[0]) * count, $array_ref, 0, NULL, NULL ); - if (err != CL_SUCCESS) - { - printf("Error: Failed to read data array! %d\n", err); - return err; - } - - // Shutdown and cleanup - // - clReleaseMemObject(device_data); - clReleaseProgram(program); - clReleaseKernel(kernel); - clReleaseCommandQueue(commands); - clReleaseContext(context); - - return 0; -} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..830e30f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +numpy +pygments diff --git a/setup.py b/setup.py index dc7191c..4fa8a79 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def visit(destination_directory, source_directory): setup( name='ctree', - version='0.95a', + version='0.1.9', description='A C-family AST implementation designed to be an IR for DSL compilers.', packages=[ @@ -56,12 +56,15 @@ def visit(destination_directory, source_directory): 'ctree.ocl', 'ctree.omp', 'ctree.py', + 'ctree.np', 'ctree.simd', 'ctree.templates', 'ctree.opentuner', 'ctree.metrics', 'ctree.tools', 'ctree.tools.generators', + 'ctree.tools.generators.templates', + 'ctree.transforms', 'ctree.visual', ], @@ -71,9 +74,7 @@ def visit(destination_directory, source_directory): install_requires=[ 'numpy', - 'mako', - 'pyserial', - # 'readline', + 'pyserial' ], data_files=data_file_list, diff --git a/test/fixtures/sample_asts.py b/test/fixtures/sample_asts.py index 5be6e98..8777012 100644 --- a/test/fixtures/sample_asts.py +++ b/test/fixtures/sample_asts.py @@ -2,9 +2,20 @@ A collection of pre-built ASTs for use in testing. """ +from ctypes import * + from ctree.c.nodes import * -from ctree.c.types import * from ctree.cpp.nodes import * +import ctree.np + +ctree.np # Make PEP8 Happy + + +# --------------------------------------------------------------------------- +# all sample ASTs in a list for iteration. ASTs must add themselves. + +SAMPLE_ASTS = [] + # --------------------------------------------------------------------------- # integer identity @@ -15,14 +26,17 @@ def identity(x): identity_ast = \ - FunctionDecl(Int(), "identity", [SymbolRef(SymbolRef("x"), Int())], [ + FunctionDecl(c_int(), "identity", [SymbolRef(SymbolRef("x"), c_int())], [ Return(SymbolRef("x")) ]) +SAMPLE_ASTS.append((identity, identity_ast)) + # --------------------------------------------------------------------------- # greatest common divisor + def gcd(a, b): if b == 0: return a @@ -31,17 +45,21 @@ def gcd(a, b): gcd_ast = \ - FunctionDecl(Int(), "gcd", [SymbolRef("a", Int()), SymbolRef("b", Int())], [ - If(Eq(SymbolRef('b'), Constant(0)), - [Return(SymbolRef('a'))], - [Return(FunctionCall(SymbolRef('gcd'), [SymbolRef('b'), Mod(SymbolRef('a'), - SymbolRef('b'))]))]) - ]) + FunctionDecl(c_int(), "gcd", + [SymbolRef("a", c_int()), SymbolRef("b", c_int())], [ + If(Eq(SymbolRef('b'), Constant(0)), + [Return(SymbolRef('a'))], + [Return(FunctionCall(SymbolRef('gcd'), + [SymbolRef('b'), Mod(SymbolRef('a'), + SymbolRef('b'))]))]) + ]) +SAMPLE_ASTS.append((gcd, gcd_ast)) # --------------------------------------------------------------------------- # naive fibonacci + def fib(n): if n < 2: return n @@ -50,30 +68,36 @@ def fib(n): fib_ast = \ - FunctionDecl(Int(), "fib", [SymbolRef("n", Int())], [ + FunctionDecl(c_int(), "fib", [SymbolRef("n", c_int())], [ If(Lt(SymbolRef("n"), Constant(2)), [Return(SymbolRef("n"))], - [Return(Add(FunctionCall(SymbolRef("fib"), [Sub(SymbolRef("n"), Constant(1))]), - FunctionCall(SymbolRef("fib"), [Sub(SymbolRef("n"), Constant(2))])))]) + [Return(Add(FunctionCall(SymbolRef("fib"), + [Sub(SymbolRef("n"), Constant(1))]), + FunctionCall(SymbolRef("fib"), + [Sub(SymbolRef("n"), Constant(2))])))]) ]) +SAMPLE_ASTS.append((fib, fib_ast)) # --------------------------------------------------------------------------- # a zero-argument function + def get_two(): return 2 get_two_ast = \ - FunctionDecl(Long(), "get_two", [], [ + FunctionDecl(c_long(), "get_two", [], [ Return(Constant(2)) ]) +SAMPLE_ASTS.append((get_two, get_two_ast)) # --------------------------------------------------------------------------- # a function with mixed argument types + def choose(p, a, b): if p < 0.5: return a @@ -82,15 +106,17 @@ def choose(p, a, b): choose_ast = \ - FunctionDecl(Long(), "choose", - [SymbolRef("p", Double()), SymbolRef("a", Long()), SymbolRef("b", Long())], [ - If(Lt(SymbolRef("p"), Constant(0.5)), [ - Return(SymbolRef("a")), - ], [ - Return(SymbolRef("b")), - ]) - ]) - + FunctionDecl(c_long(), "choose", + [SymbolRef("p", c_double()), + SymbolRef("a", c_long()), + SymbolRef("b", c_long())], + [ + If(Lt(SymbolRef("p"), Constant(0.5)), + [Return(SymbolRef("a"))], + [Return(SymbolRef("b"))]) + ]) + +SAMPLE_ASTS.append((choose, choose_ast)) # --------------------------------------------------------------------------- # a function that takes a numpy array @@ -98,25 +124,33 @@ def choose(p, a, b): import math import numpy as np + def l2norm(A): - return math.sqrt(sum(x*x for x in A)) + return np.sqrt(np.sum(np.square(A))) l2norm_ast = CFile("generated", [ CppInclude("math.h"), - FunctionDecl(Double(), "l2norm", - params=[ - SymbolRef("A", NdPointer(np.float64, 1, 12)), - SymbolRef("n", Int()), - ], - defn=[ - SymbolRef("sum", Double()), - For(Assign(SymbolRef("i", Int()), Constant(0)), - Lt(SymbolRef("i"), SymbolRef("n")), - PostInc(SymbolRef("i")), [ - AddAssign(SymbolRef("sum"), - Mul(ArrayRef(SymbolRef("A"), SymbolRef("i")), - ArrayRef(SymbolRef("A"), SymbolRef("i")))), - ]), - Return( FunctionCall("sqrt", [SymbolRef("sum")]) ), - ]) + FunctionDecl(c_double(), "l2norm", + params=[ + SymbolRef("A", + np.ctypeslib.ndpointer( + dtype=np.float64, ndim=1, shape=(12,) + )()), + SymbolRef("n", c_int()), + ], + defn=[ + Assign(SymbolRef("sum", c_double()), Constant(0)), + For(Assign(SymbolRef("i", c_int()), Constant(0)), + Lt(SymbolRef("i"), SymbolRef("n")), + PostInc(SymbolRef("i")), [ + AddAssign(SymbolRef("sum"), + Mul(ArrayRef(SymbolRef("A"), + SymbolRef("i")), + ArrayRef(SymbolRef("A"), + SymbolRef("i")))), + ]), + Return(FunctionCall("sqrt", [SymbolRef("sum")])), + ]) ]) + +SAMPLE_ASTS.append((l2norm, l2norm_ast)) diff --git a/test/test_ArrayDefs.py b/test/test_ArrayDefs.py index f33411c..b0c673b 100644 --- a/test/test_ArrayDefs.py +++ b/test/test_ArrayDefs.py @@ -1,21 +1,24 @@ -import unittest +import ctypes as ct -from ctree.c.nodes import * +from util import CtreeTest +from ctree.c.nodes import SymbolRef, Constant, Add, Mul, ArrayDef, Sub, Array -class TestArrayDefs(unittest.TestCase): +class TestArrayDefs(CtreeTest): def test_simple_array_def(self): - self.assertEqual(str(ArrayDef([Constant(0), Constant(1)])), "{ 0, 1 }") + self._check_code(ArrayDef( + SymbolRef('hi', ct.c_int()), Constant(2), + Array(body=[Constant(0), Constant(1)]), + ), "int hi[2] = {0, 1}") def test_complex(self): - node = Assign( - SymbolRef('myArray'), - ArrayDef( - [ - Add(SymbolRef('b'), SymbolRef('c')), - Mul(Sub(Constant(99), SymbolRef('d')), Constant(200)) - ] - ) + node = ArrayDef( + SymbolRef('myArray', ct.c_int()), + Constant(2), + Array(body=[ + Add(SymbolRef('b'), SymbolRef('c')), + Mul(Sub(Constant(99), SymbolRef('d')), Constant(200)) + ]) ) - self.assertEqual(str(node), "myArray = { b + c, (99 - d) * 200 }") + self._check_code(node, "int myArray[2] = {b + c, (99 - d) * 200}") diff --git a/test/test_analyses.py b/test/test_analyses.py index 0b2861a..b6f6720 100644 --- a/test/test_analyses.py +++ b/test/test_analyses.py @@ -1,37 +1,10 @@ import unittest -from ctree.c.nodes import * from ctree.analyses import * from ctree.frontend import get_ast from fixtures.sample_asts import * -class TestVerifyParentPointers(unittest.TestCase): - def test_identity(self): - VerifyParentPointers().visit(identity_ast) - - def test_fib(self): - VerifyParentPointers().visit(fib_ast) - - def test_gcd(self): - VerifyParentPointers().visit(gcd_ast) - - def test_raise_identity(self): - identity_ast.find(SymbolRef, name="x").parent = None - with self.assertRaises(AstValidationError): - VerifyParentPointers().visit(identity_ast) - - def test_raise_fib(self): - fib_ast.find(Constant, value=2).parent = None - with self.assertRaises(AstValidationError): - VerifyParentPointers().visit(fib_ast) - - def test_raise_gcd(self): - gcd_ast.find(Return).parent = None - with self.assertRaises(AstValidationError): - VerifyParentPointers().visit(gcd_ast) - - class TestVerifyOnlyCtreeNodes(unittest.TestCase): def _check(self, tree): VerifyOnlyCtreeNodes().visit(tree) diff --git a/test/test_assign.py b/test/test_assign.py index 046713b..511aa11 100644 --- a/test/test_assign.py +++ b/test/test_assign.py @@ -1,5 +1,7 @@ import unittest +import ast +from ctree.transformations import PyBasicConversions from ctree.c.nodes import * @@ -10,3 +12,28 @@ def setUp(self): def test_simple_assign(self): node = Assign(self.foo, self.bar) self.assertEqual(str(node), "foo = bar") + + + def test_multiple_assign_simple(self): + node = ast.Assign([ast.Tuple(elts = (ast.Name(id = "x", ctx = None), ast.Name(id = "y", ctx = None)))], ast.Tuple(elts = (ast.Name(id = "x", ctx = None), ast.Name(id = "y", ctx = None)))) + transformed_node = PyBasicConversions().visit(node) + + self.assertEqual(str(transformed_node), "\n____temp__x = x;\n____temp__y = y;\nx = ____temp__x;\ny = ____temp__y;\n") + + def test_multiple_assign_constant(self): + node = ast.Assign([ast.Tuple(elts = (ast.Name(id = "x", ctx = None), ast.Name(id = "y", ctx = None)))], ast.Tuple(elts = (Constant(1), Constant(2)))) + transformed_node = PyBasicConversions().visit(node) + + self.assertEqual(str(transformed_node), "\nx = 1;\ny = 2;\n") + + def test_multiple_assign_dependent(self): + node = ast.Assign([ast.Tuple(elts = (ast.Name(id = "x", ctx = None), ast.Name(id = "y", ctx = None)))], ast.Tuple(elts = (ast.Name(id = "y", ctx = None), ast.Name(id = "x", ctx = None)))) + transformed_node = PyBasicConversions().visit(node) + + self.assertEqual(str(transformed_node), "\n____temp__x = x;\n____temp__y = y;\ny = ____temp__y;\nx = ____temp__x;\n") + + def test_multiple_assign_dependent(self): + node = ast.Assign([ast.Tuple(elts = (ast.Name(id = "x", ctx = None), ast.Name(id = "y", ctx = None)))], ast.Tuple(elts = (FunctionCall(func = 'square', args = [Constant(5), Constant(5)]), FunctionCall(func = 'square', args = [Constant(5), Constant(5)])))) + transformed_node = PyBasicConversions().visit(node) + + self.assertEqual(str(transformed_node), "\n____temp__x = square(5, 5);\n____temp__y = square(5, 5);\nx = ____temp__x;\ny = ____temp__y;\n") diff --git a/test/test_casts.py b/test/test_casts.py index c377b8d..e0fbeab 100644 --- a/test/test_casts.py +++ b/test/test_casts.py @@ -1,25 +1,21 @@ -import unittest +from ctypes import * +from util import CtreeTest from ctree.c.nodes import * -from ctree.c.types import * -class TestCastOps(unittest.TestCase): +class TestCastOps(CtreeTest): def setUp(self): self.foo = SymbolRef('foo') - def _check(self, tree, expected): - actual = str(tree) - self.assertEqual(actual, expected) - def test_void(self): - tree = Cast(Ptr(Void()), self.foo) - self._check(tree, "(void*) foo") + tree = Cast(c_void_p(), self.foo) + self._check_code(tree, "(void*) foo") def test_int(self): - tree = Cast(Int(), self.foo) - self._check(tree, "(int) foo") + tree = Cast(c_int(), self.foo) + self._check_code(tree, "(int) foo") def test_int_p(self): - tree = Cast(Ptr(Int()), self.foo) - self._check(tree, "(int*) foo") + tree = Cast(POINTER(c_int)(), self.foo) + self._check_code(tree, "(int*) foo") diff --git a/test/test_compare.py b/test/test_compare.py new file mode 100644 index 0000000..bcb6a29 --- /dev/null +++ b/test/test_compare.py @@ -0,0 +1,32 @@ +import unittest +import ast +from ctree.c.nodes import BinaryOp +from ctree.transformations import PyBasicConversions + + +class TestCompare(unittest.TestCase): + + def test_LessThan(self): + comp = ast.parse("5 < foo < 6") + comp = PyBasicConversions().visit(comp).find(BinaryOp) + self.assertEqual(str(comp), "5 < foo && foo < 6") + + def test_LessThanEqual(self): + comp = ast.parse("5 <= foo <= 6") + comp = PyBasicConversions().visit(comp).find(BinaryOp) + self.assertEqual(str(comp), "5 <= foo && foo <= 6") + + def test_GreaterThan(self): + comp = ast.parse("5 > foo > 6") + comp = PyBasicConversions().visit(comp).find(BinaryOp) + self.assertEqual(str(comp), "5 > foo && foo > 6") + + def test_GreaterThan(self): + comp = ast.parse("5 >= foo >= 6") + comp = PyBasicConversions().visit(comp).find(BinaryOp) + self.assertEqual(str(comp), "5 >= foo && foo >= 6") + + def test_Equals(self): + comp = ast.parse("5 == foo == 6") + comp = PyBasicConversions().visit(comp).find(BinaryOp) + self.assertEqual(str(comp), "5 == foo && foo == 6") \ No newline at end of file diff --git a/test/test_ctree_nodes.py b/test/test_ctree_nodes.py new file mode 100644 index 0000000..fb51a03 --- /dev/null +++ b/test/test_ctree_nodes.py @@ -0,0 +1,26 @@ +import unittest +import ctypes as ct + +from ctree.c.nodes import * + + +class TestCtreeNode(unittest.TestCase): + + def test_bad_override_codegen(self): + class BadNode(CtreeNode): + pass + with self.assertRaises(Exception): + BadNode().codegen() + + def test_bad_override_to_dot(self): + class BadNode(CtreeNode): + pass + with self.assertRaises(Exception): + BadNode()._to_dot() + + def test_find_all_attr_error(self): + tree = Add(SymbolRef('a'), Constant(10)) + try: + tree.find_all(SymbolRef, type=ct.c_int()) + except AttributeError: + self.fail("find_all should not raise AttributeError") diff --git a/test/test_decls.py b/test/test_decls.py index 4774475..26c6360 100644 --- a/test/test_decls.py +++ b/test/test_decls.py @@ -1,18 +1,14 @@ -import unittest +from ctypes import * +from util import CtreeTest from ctree.c.nodes import * -from ctree.c.types import * -class TestVarDecls(unittest.TestCase): - def _check(self, tree, expected): - actual = str(tree) - self.assertEqual(actual, expected) - +class TestVarDecls(CtreeTest): def test_simple_00(self): - foo = SymbolRef('foo', sym_type=Int()) - self._check(foo, "int foo") + tree = SymbolRef('foo', c_double()) + self._check_code(tree, "double foo") def test_simple_01(self): - foo = Assign(SymbolRef('foo', sym_type=Double()), Constant(1.2)) - self._check(foo, "double foo = 1.2") + tree = Assign(SymbolRef('foo', c_double()), Constant(1.2)) + self._check_code(tree, "double foo = 1.2") diff --git a/test/test_dot.py b/test/test_dot.py index 04be7d7..29f8f6d 100644 --- a/test/test_dot.py +++ b/test/test_dot.py @@ -1,6 +1,5 @@ import unittest -from ctree.dotgen import to_dot from ctree.frontend import get_ast from fixtures.sample_asts import * @@ -14,22 +13,22 @@ class TestDotGen(unittest.TestCase): """ def test_c_identity(self): - self.assertNotEqual(to_dot(identity_ast), "") + self.assertNotEqual(identity_ast.to_dot(), "") def test_c_gcd(self): - self.assertNotEqual(to_dot(gcd_ast), "") + self.assertNotEqual(gcd_ast.to_dot(), "") def test_c_fib(self): - self.assertNotEqual(to_dot(fib_ast), "") + self.assertNotEqual(fib_ast.to_dot(), "") def test_c_l2norm(self): - self.assertNotEqual(to_dot(l2norm_ast), "") + self.assertNotEqual(l2norm_ast.to_dot(), "") def test_py_identity(self): - self.assertNotEqual(to_dot(get_ast(identity)), "") + self.assertNotEqual(get_ast(identity).to_dot(), "") def test_py_gcd(self): - self.assertNotEqual(to_dot(get_ast(gcd)), "") + self.assertNotEqual(get_ast(gcd).to_dot(), "") def test_py_fib(self): - self.assertNotEqual(to_dot(get_ast(fib)), "") + self.assertNotEqual(get_ast(fib).to_dot(), "") diff --git a/test/test_dot_manager.py b/test/test_dot_manager.py new file mode 100644 index 0000000..9fcbc39 --- /dev/null +++ b/test/test_dot_manager.py @@ -0,0 +1,22 @@ +__author__ = 'Chick Markley' + +import unittest + +from ctree.visual.dot_manager import DotManager +from ctree.frontend import get_ast + + +def square_of(n): + return n * n + +class TestDotManager(unittest.TestCase): + """ + Difficult to test because of ipython and dot dependencies + """ + + @unittest.skip("difficult to test because of ipython and dot dependencies") + def test_c_identity(self): + tree = get_ast(square_of) + DotManager.run_dot(tree.to_dot()) + + diff --git a/test/test_examples.py b/test/test_examples.py index 594fbc5..a0400e3 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -5,9 +5,10 @@ """ import unittest +import ctree try: - import examples.ArrayDoubler + import examples except ImportError: HAVE_EXAMPLES = False else: @@ -40,7 +41,7 @@ def test_TuningSpecializer(self): from examples import TuningSpecializer TuningSpecializer.main() - @unittest.skip("intermitten failures") + @unittest.skipUnless(ctree.OCL_ENABLED, "OpenCL mode not enabled") def test_OclDoubler(self): from examples import OclDoubler OclDoubler.main() diff --git a/test/test_file.py b/test/test_file.py index cd3ca62..3987ac5 100644 --- a/test/test_file.py +++ b/test/test_file.py @@ -1,20 +1,16 @@ -import unittest +from ctypes import * +from util import CtreeTest from ctree.c.nodes import * -from ctree.c.types import * -class TestFile(unittest.TestCase): - def _check(self, tree, expected): - actual = str(tree) - self.assertEqual(actual, expected) - +class TestFile(CtreeTest): def test_simple_00(self): - foo = SymbolRef("foo", sym_type=Int()) - bar = FunctionDecl(Float(), SymbolRef("bar")) + foo = SymbolRef("foo", sym_type=c_int()) + bar = FunctionDecl(c_double(), SymbolRef("bar")) tree = CFile("myfile", [foo, bar]) - self._check(tree, """\ -// -int foo; -float bar(); -""") + self._check_code(tree, """\ + // + int foo; + double bar(); + """) diff --git a/test/test_flattening.py b/test/test_flattening.py index fe5eb54..978842d 100644 --- a/test/test_flattening.py +++ b/test/test_flattening.py @@ -3,9 +3,6 @@ from ctree.c.nodes import * from ctree.analyses import * -from ctree.frontend import get_ast -from ctree.dotgen import to_dot - from ctree.util import flatten, enumerate_flatten @@ -107,4 +104,4 @@ def test_lolol_3(self): class TestFlatteningDotGen(unittest.TestCase): def test_lol_1(self): tree = Block([[a, b]]) - to_dot(tree) + tree.to_dot() diff --git a/test/test_frontend.py b/test/test_frontend.py index b72eb57..7d7c429 100644 --- a/test/test_frontend.py +++ b/test/test_frontend.py @@ -1,7 +1,7 @@ -import ast import unittest +from inspect import getsource -from ctree.frontend import get_ast +from ctree.frontend import * from fixtures.sample_asts import * @@ -14,3 +14,10 @@ def test_gcd(self): def test_fib(self): self.assertIsInstance(get_ast(fib), ast.AST) + + def test_parse_print(self): + parseprint(getsource(fib)) + + def test_dump(self): + dump(fib_ast) + #self.assertEqual(dump(fib_ast), 'FunctionDecl(params=[\n SymbolRef(),\n ], defn=[\n If(cond=BinaryOp(left=SymbolRef(), right=Constant()), then=[\n Return(value=SymbolRef()),\n ], elze=[\n Return(value=BinaryOp(left=FunctionCall(func=SymbolRef(), args=[\n BinaryOp(left=SymbolRef(), right=Constant()),\n ]), right=FunctionCall(func=SymbolRef(), args=[\n BinaryOp(left=SymbolRef(), right=Constant()),\n ]))),\n ]),\n ])') \ No newline at end of file diff --git a/test/test_funcdecls.py b/test/test_funcdecls.py index bd2e5ea..97d09ce 100644 --- a/test/test_funcdecls.py +++ b/test/test_funcdecls.py @@ -1,45 +1,48 @@ -import unittest +from ctypes import * +from util import CtreeTest from ctree.c.nodes import * -from ctree.c.types import * -class TestFuncDecls(unittest.TestCase): - def _check(self, tree, expected): - actual = str(tree) - self.assertEqual(actual, expected) - +class TestFuncDecls(CtreeTest): def test_voidvoid(self): - node = FunctionDecl(Ptr(Void()), SymbolRef("foo")) - self._check(node, "void* foo()") + node = FunctionDecl(c_void_p(), SymbolRef("foo")) + self._check_code(node, "void* foo()") def test_intvoid(self): - node = FunctionDecl(Int(), SymbolRef("foo")) - self._check(node, "int foo()") + node = FunctionDecl(c_int(), SymbolRef("foo")) + self._check_code(node, "int foo()") def test_voidint(self): - params = [SymbolRef("a", Int())] - node = FunctionDecl(Ptr(Void()), SymbolRef("foo"), params) - self._check(node, "void* foo(int a)") + params = [SymbolRef("a", c_int())] + node = FunctionDecl(c_void_p(), SymbolRef("foo"), params) + self._check_code(node, "void* foo(int a)") def test_intint(self): - params = [SymbolRef("b", Int())] - node = FunctionDecl(Int(), SymbolRef("foo"), params) - self._check(node, "int foo(int b)") + params = [SymbolRef("b", c_int())] + node = FunctionDecl(c_int(), SymbolRef("foo"), params) + self._check_code(node, "int foo(int b)") def test_voidintint(self): - params = [SymbolRef("c", Int()), SymbolRef("d", Int())] - node = FunctionDecl(Ptr(Void()), SymbolRef("foo"), params) - self._check(node, "void* foo(int c, int d)") + params = [SymbolRef("c", c_int()), SymbolRef("d", c_int())] + node = FunctionDecl(c_void_p(), SymbolRef("foo"), params) + self._check_code(node, "void* foo(int c, int d)") def test_voidintint_names(self): - params = [SymbolRef("bar", Int()), SymbolRef('baz', Int())] - node = FunctionDecl(Ptr(Void()), SymbolRef("foo"), params) - self._check(node, "void* foo(int bar, int baz)") + params = [SymbolRef("bar", c_int()), SymbolRef('baz', c_int())] + node = FunctionDecl(c_void_p(), SymbolRef("foo"), params) + self._check_code(node, "void* foo(int bar, int baz)") def test_withdefn(self): body = [Add(SymbolRef('foo'), SymbolRef('bar'))] - node = FunctionDecl(Ptr(Void()), SymbolRef("fn"), defn=body) - self._check(node, """void* fn() { - foo + bar; -}""") + node = FunctionDecl(c_void_p(), SymbolRef("fn"), defn=body) + self._check_code(node, """\ + void* fn() { + foo + bar; + }""") + + def test_set_kernel(self): + params = [SymbolRef("bar", c_int()), SymbolRef('baz', c_int())] + node = FunctionDecl(c_void_p(), SymbolRef("foo"), params) + node.set_kernel(); + self._check_code(node, "__kernel void* foo(int bar, int baz)") diff --git a/test/test_highlight.py b/test/test_highlight.py index 761bf88..5bbd478 100644 --- a/test/test_highlight.py +++ b/test/test_highlight.py @@ -12,6 +12,14 @@ def test_no_pygments(self): self.assertEqual(code, highlighted) +PYGMENTS_INSTALLED = True +try: + import pygments +except: + PYGMENTS_INSTALLED = False + + +@unittest.skipUnless(PYGMENTS_INSTALLED, "Skipping pygments tests") class TestHighlights(unittest.TestCase): def test_highlight_c(self): highlighted = highlight("int a = 0;", "c") diff --git a/test/test_import.py b/test/test_import.py index 3707606..f1d0faa 100644 --- a/test/test_import.py +++ b/test/test_import.py @@ -1,6 +1,5 @@ import unittest - class TestImport(unittest.TestCase): def test_import_base(self): import ctree diff --git a/test/test_inserts.py b/test/test_inserts.py deleted file mode 100644 index d4d0ecc..0000000 --- a/test/test_inserts.py +++ /dev/null @@ -1,89 +0,0 @@ -import unittest -from textwrap import dedent - -from ctree.c.nodes import * - - -class TestAstInsertion(unittest.TestCase): - def setUp(self): - self.front = SymbolRef("a") - self.mid = SymbolRef("b") - self.back = SymbolRef("c") - self.block = Block([ - self.front, - self.mid, - self.back, - ]) - - def _check(self, tree, expected_i): - actual = str(tree) - expected = dedent(expected_i) - self.assertEqual(actual, expected) - - def test_insert_before_front(self): - self.front.insert_before(SymbolRef("d")) - self._check(self.block, """\ - { - d; - a; - b; - c; - }""") - - def test_insert_after_front(self): - self.front.insert_after(SymbolRef("d")) - self._check(self.block, """\ - { - a; - d; - b; - c; - }""") - - def test_insert_before_mid(self): - self.mid.insert_before(SymbolRef("d")) - self._check(self.block, """\ - { - a; - d; - b; - c; - }""") - - def test_insert_after_mid(self): - self.mid.insert_after(SymbolRef("d")) - self._check(self.block, """\ - { - a; - b; - d; - c; - }""") - - def test_insert_before_back(self): - self.back.insert_before(SymbolRef("d")) - self._check(self.block, """\ - { - a; - b; - d; - c; - }""") - - def test_insert_after_back(self): - self.back.insert_after(SymbolRef("d")) - self._check(self.block, """\ - { - a; - b; - c; - d; - }""") - - def test_bad_insert_before(self): - with self.assertRaises(Exception): - self.block.insert_before(SymbolRef("d")) - - def test_bad_insert_after(self): - with self.assertRaises(Exception): - self.block.insert_after(SymbolRef("d")) diff --git a/test/test_jit.py b/test/test_jit.py index 7ce2f8e..f4d452a 100644 --- a/test/test_jit.py +++ b/test/test_jit.py @@ -1,48 +1,76 @@ import unittest from ctree.jit import * +from ctree import CONFIG from fixtures.sample_asts import * +import ctypes +from ctree.transformations import PyBasicConversions + +class TestTranslator(LazySpecializedFunction): + def args_to_subconfig(self, args): + return {'arg_typesig': tuple(type(get_ctype(a)) for a in args)} + + def transform(self, tree, program_config): + arg_types = program_config[0]['arg_typesig'] + tree = PyBasicConversions().visit(tree.body[0]) + tree.return_type = arg_types[0]() + for param, ty in zip(tree.params, arg_types): + param.type = ty() + return [CFile(tree.name, [tree])] + + def finalize(self, transform_result, program_config): + proj = Project(transform_result) + cfile = transform_result[0] + arg_types = program_config[0]['arg_typesig'] + + func_type = ctypes.CFUNCTYPE(arg_types[0], *arg_types) + return BasicFunction(cfile.name, proj, func_type) + +class BasicFunction(ConcreteSpecializedFunction): + def __init__(self, entry, tree, typesig): + self._c_function = self._compile(entry, tree, typesig) + + def __call__(self, *args, **kwargs): + return self._c_function(*args, **kwargs) class TestJit(unittest.TestCase): def test_identity(self): mod = JitModule() - submod = CFile("generated", [identity_ast]). \ - _compile(identity_ast.codegen(), mod.compilation_dir) + submod = CFile("test_identity", [identity_ast], path=CONFIG.get('jit','COMPILE_PATH')). \ + _compile(identity_ast.codegen()) mod._link_in(submod) c_identity_fn = mod.get_callable(identity_ast.name, - identity_ast.get_type().as_ctype()) + identity_ast.get_type()) self.assertEqual(identity(1), c_identity_fn(1)) self.assertEqual(identity(12), c_identity_fn(12)) self.assertEqual(identity(123), c_identity_fn(123)) def test_fib(self): mod = JitModule() - submod = CFile("generated", [fib_ast])._compile(fib_ast.codegen(), - mod.compilation_dir) + submod = CFile("test_fib", [fib_ast], path=CONFIG.get('jit','COMPILE_PATH'))._compile(fib_ast.codegen()) mod._link_in(submod) c_fib_fn = mod.get_callable(fib_ast.name, - fib_ast.get_type().as_ctype()) + fib_ast.get_type()) self.assertEqual(fib(1), c_fib_fn(1)) self.assertEqual(fib(6), c_fib_fn(6)) def test_gcd(self): mod = JitModule() - submod = CFile("generated", [gcd_ast])._compile(gcd_ast.codegen(), - mod.compilation_dir) + submod = CFile("test_gcd", [gcd_ast], path=CONFIG.get('jit','COMPILE_PATH'))._compile(gcd_ast.codegen()) mod._link_in(submod) c_gcd_fn = mod.get_callable(gcd_ast.name, - gcd_ast.get_type().as_ctype()) + gcd_ast.get_type()) self.assertEqual(gcd(44, 122), c_gcd_fn(44, 122)) self.assertEqual(gcd(27, 39), c_gcd_fn(27, 39)) def test_choose(self): mod = JitModule() - submod = CFile("generated", [choose_ast]). \ - _compile(choose_ast.codegen(), mod.compilation_dir) + submod = CFile("test_choose", [choose_ast], path=CONFIG.get('jit','COMPILE_PATH')). \ + _compile(choose_ast.codegen()) mod._link_in(submod) c_choose_fn = mod.get_callable(choose_ast.name, - choose_ast.get_type().as_ctype()) + choose_ast.get_type()) self.assertEqual(choose(0.2, 44, 122), c_choose_fn(0.2, 44, 122)) self.assertEqual(choose(0.8, 44, 122), c_choose_fn(0.8, 44, 122)) self.assertEqual(choose(0.3, 27, 39), c_choose_fn(0.3, 27, 39)) @@ -50,11 +78,29 @@ def test_choose(self): def test_l2norm(self): mod = JitModule() - submod = CFile("generated", - [l2norm_ast])._compile(l2norm_ast.codegen(), - mod.compilation_dir) + submod = CFile("test_l2norm", + [l2norm_ast], path=CONFIG.get('jit','COMPILE_PATH'))._compile(l2norm_ast.codegen()) mod._link_in(submod) entry = l2norm_ast.find(FunctionDecl, name="l2norm") - c_l2norm_fn = mod.get_callable(entry.name, entry.get_type().as_ctype()) + c_l2norm_fn = mod.get_callable(entry.name, entry.get_type()) self.assertEqual(l2norm(np.ones(12, dtype=np.float64)), c_l2norm_fn(np.ones(12, dtype=np.float64), 12)) + + def test_getFile(self): + getFile(os.path.join(CONFIG.get('jit','COMPILE_PATH'),'test_l2norm.c')) + +class TestAuxiliary(unittest.TestCase): + def test_NameExtractor(self): + def f(x): + return x + 3 + + py_ast = get_ast(f) + result = LazySpecializedFunction.NameExtractor().visit(py_ast) + self.assertEqual(result, 'f') + + def test_from_function(self): + def f(x): + return x + 3 + + c_f = TestTranslator.from_function(f, 'test_from_function') + self.assertEqual(c_f(3), 6) diff --git a/test/test_lambda.py b/test/test_lambda.py new file mode 100644 index 0000000..b57830a --- /dev/null +++ b/test/test_lambda.py @@ -0,0 +1,68 @@ +import unittest +import ctypes as ct +import ast +import sys + +from ctree.transformations import PyBasicConversions +from ctree.transforms import DeclarationFiller +from ctree.c.nodes import * + + +class TestAssigns(unittest.TestCase): + + + def mini_transform(self, node): + """ + This method acts as a simulation of a specializer's transform() method. It's the bare minimum required of + a transform() method by the specializer writer. + + :param node: the node to transform + :return: the node transformed through PyBasicConversions into a rough C-AST. + """ + transformed_node = PyBasicConversions().visit(node) + + transformed_node.name = "apply" + transformed_node.return_type = ct.c_float() + + for param in transformed_node.params: + param.type = ct.c_float() + + return transformed_node + + def mini__call__(self, node): + """ + This method acts as a simulation of jit.py's __call__() method. The specializer writer does not have to write + this method. + + :param node: the node to generate code for + :return: a type-complete C-AST corresponding to the input node + """ + transformed_node = self.mini_transform(node) + return DeclarationFiller().visit(transformed_node) + + @unittest.skipIf(sys.version_info >= (3,0), 'Lambdas changed in py3k') + def test_one_arg_lambda(self): + """ + This method tests the squaring lambda function, a one argument lambda function. + """ + square_lambda_node = ast.Lambda(args = ast.arguments([SymbolRef("x")], None, None, None), body = Mul(SymbolRef("x"), SymbolRef("x"))) + + # simulating __call__() + type_inferred_node = self.mini__call__(square_lambda_node) + + self.assertEqual(str(type_inferred_node), "float apply(float x) {\n" + \ + " return x * x;\n}") + + + @unittest.skipIf(sys.version_info >= (3,0), 'Lambdas changed in py3k') + def test_two_arg_lambda(self): + """ + This method tests the adding lambda function, a two argument lambda function. + """ + add_lambda_node = ast.Lambda(args = ast.arguments([SymbolRef("x"), SymbolRef("y")], None, None, None), body = Add(SymbolRef("x"), SymbolRef("y"))) + + # simulating __call__() + type_inferred_node = self.mini__call__(add_lambda_node) + + self.assertEqual(str(type_inferred_node), "float apply(float x, float y) {\n" + \ + " return x + y;\n}") diff --git a/test/test_lifter.py b/test/test_lifter.py new file mode 100644 index 0000000..71416a6 --- /dev/null +++ b/test/test_lifter.py @@ -0,0 +1,102 @@ +from copy import deepcopy + +from util import CtreeTest +from fixtures.sample_asts import * +from ctree.transformations import Lifter +import sys + +class TestLifter(CtreeTest): + def test_nop(self): + for py_fn, tree in SAMPLE_ASTS: + transformed = Lifter().visit(deepcopy(tree)) + self._check_code(actual=tree, expected=transformed) + + def test_one_param(self): + inner = SymbolRef("foo") + inner.lift(params=[SymbolRef(inner.name, c_double())]) + + tree = FunctionDecl(None, "fn", [], [ + Assign(inner, Constant(123.0)), + ]) + + tree = Lifter().visit(tree) + + self._check_code(actual=tree, expected="""\ + void fn(double foo) { + foo = 123.0; + }""") + + def test_two_params(self): + inner0 = SymbolRef("foo") + inner0.lift(params=[SymbolRef(inner0.name, c_int())]) + + inner1 = SymbolRef("bar") + inner1.lift(params=[SymbolRef(inner1.name, c_double())]) + + tree = FunctionDecl(None, "fn", [], [ + Assign(inner0, Constant(123)), + Assign(inner1, Constant(456.7)), + ]) + + tree = Lifter().visit(tree) + + self._check_code(actual=tree, expected="""\ + void fn(int foo, double bar) { + foo = 123; + bar = 456.7; + }""") + + def test_one_include(self): + tree = CFile("generated", [deepcopy(get_two_ast)]) + stmt = tree.find(FunctionDecl).defn[0] + stmt.lift(includes=[CppInclude("stdio.h")]) + + tree = Lifter().visit(tree) + + if sys.maxsize > 2 ** 32: + self._check_code(actual=tree, expected="""\ + // + #include + long get_two() { + return 2; + }; + """) + else: + self._check_code(actual=tree, expected="""\ + // + #include + int get_two() { + return 2; + }; + """) + + def test_multi_includes(self): + tree = CFile("generated", [deepcopy(get_two_ast)]) + stmt0 = tree.find(FunctionDecl) + stmt1 = stmt0.defn[0] + + stmt0.lift(includes=[CppInclude("stdio.h")]) + stmt1.lift(includes=[CppInclude("stdlib.h"), CppInclude("float.h")]) + + tree = Lifter().visit(tree) + + if sys.maxsize > 2 ** 32: + self._check_code(actual=tree, expected="""\ + // + #include + #include + #include + long get_two() { + return 2; + }; + """) + else: + self._check_code(actual=tree, expected="""\ + // + #include + #include + #include + int get_two() { + return 2; + }; + """) diff --git a/test/test_list.py b/test/test_list.py new file mode 100644 index 0000000..19f2cd1 --- /dev/null +++ b/test/test_list.py @@ -0,0 +1,13 @@ +__author__ = 'dorthyluu' + +import unittest +import ast +from ctree.transformations import PyBasicConversions +from ctree.c.nodes import Array + +class TestList(unittest.TestCase): + + def test_List(self): + array = ast.parse("[1, 5, 7, 3]") + array = PyBasicConversions().visit(array).find(Array) + self.assertEqual(str(array), "{1, 5, 7, 3}") diff --git a/test/test_numpy.py b/test/test_numpy.py new file mode 100644 index 0000000..8e8fbd0 --- /dev/null +++ b/test/test_numpy.py @@ -0,0 +1,25 @@ +import _ctypes + +import numpy as np + +from ctree.types import ( + get_ctype, +) +from util import CtreeTest +from ctree.c.nodes import SymbolRef + +class TestTypeRecognizer(CtreeTest): + def test_int_array(self): + ty = get_ctype(np.arange(10, dtype=np.int32)) + self.assertIsInstance(ty, _ctypes.Array) + +class TestTypeCodeGen(CtreeTest): + def test_int_array_1d(self): + ty = get_ctype(np.arange(10, dtype=np.float32)) + tree = SymbolRef("i", ty) + self._check_code(tree, "float* i") + + def test_int_array_2d(self): + ty = get_ctype(np.arange(10, dtype=np.float32).reshape(2,5)) + tree = SymbolRef("i", ty) + self._check_code(tree, "float** i") diff --git a/test/test_ocl/test_macros.py b/test/test_ocl/test_macros.py index bfe8083..2165fc6 100644 --- a/test/test_ocl/test_macros.py +++ b/test/test_ocl/test_macros.py @@ -1,8 +1,10 @@ import unittest +import ctree from ctree.ocl.macros import * +@unittest.skipUnless(ctree.OCL_ENABLED, "OpenCL not enabled.") class TestOclMacros(unittest.TestCase): def test_CL_SUCCESS(self): tree = CL_SUCCESS() @@ -36,23 +38,69 @@ def test_barrier(self): tree = barrier(CLK_LOCAL_MEM_FENCE()) self.assertEqual(tree.codegen(), "barrier(CLK_LOCAL_MEM_FENCE)") - def get_local_id(self): + def test_get_local_id(self): tree = get_local_id(0) self.assertEqual(tree.codegen(), "get_local_id(0)") - def get_global_id(self): + def test_get_global_id(self): tree = get_global_id(0) self.assertEqual(tree.codegen(), "get_global_id(0)") - def get_local_size(self): + def test_get_group_id(self): + tree = get_group_id(0) + self.assertEqual(tree.codegen(), "get_group_id(0)") + + def test_get_local_size(self): tree = get_local_size(0) self.assertEqual(tree.codegen(), "get_local_size(0)") - def get_num_groups(self): + def test_get_num_groups(self): tree = get_num_groups(0) self.assertEqual(tree.codegen(), "get_num_groups(0)") - def clReleaseMemObject(self): + def test_clReleaseMemObject(self): tree = clReleaseMemObject(SymbolRef('device_object')) self.assertEqual(tree.codegen(), "clReleaseMemObject(device_object)") + def test_clEnqueueWriteBuffer(self): + tree = clEnqueueWriteBuffer(SymbolRef('tmp'), 'buf', False, 0, 0, + 'ptr', 0, None, None) + self.assertEqual( + tree.codegen(), + "clEnqueueWriteBuffer(tmp, buf, 0, 0, 0, ptr, 0, NULL, NULL)" + ) + + def test_clEnqueueReadBuffer(self): + tree = clEnqueueReadBuffer(SymbolRef('tmp'), 'buf', False, 0, 0, + 'ptr', 0, None, None) + self.assertEqual( + tree.codegen(), + "clEnqueueReadBuffer(tmp, buf, 0, 0, 0, ptr, 0, NULL, NULL)" + ) + + def test_clEnqueueCopyBuffer(self): + tree = clEnqueueCopyBuffer(SymbolRef('tmp'), 'a', 'b', 0, 0, 0) + self.assertEqual( + tree.codegen(), + "clEnqueueCopyBuffer(tmp, a, b, 0, 0, 0, 0, NULL, NULL)" + ) + + def test_clSetKernelArg(self): + tree = clSetKernelArg('kernel', 1, 1024, 'arg') + self.assertEqual( + tree.codegen(), + "clSetKernelArg(kernel, 1, 1024, & arg)" + ) + + def test_clEnqueueNDRangeKernel(self): + tree = clEnqueueNDRangeKernel(SymbolRef('tmp'), SymbolRef('kernel'), + 1, 0, 0, 0) + self.assertEqual( + tree.codegen(), + """{ + size_t global_size = 0; + size_t local_size = 0; + clEnqueueNDRangeKernel(tmp, kernel, 1, 0, & global_size, & local_size, 0, NULL, NULL); +}""" + ) + diff --git a/test/test_ocl/test_nodes.py b/test/test_ocl/test_nodes.py deleted file mode 100644 index b654ffb..0000000 --- a/test/test_ocl/test_nodes.py +++ /dev/null @@ -1,9 +0,0 @@ -import unittest - -from ctree.ocl.nodes import * - - -class TestOclNodes(unittest.TestCase): - def test_file(self): - f = OclFile("kernel", []) - self.assertEqual(f.codegen(), "// \n") diff --git a/test/test_ocl/test_pycl_wrapper.py b/test/test_ocl/test_pycl_wrapper.py new file mode 100644 index 0000000..4ef6d4b --- /dev/null +++ b/test/test_ocl/test_pycl_wrapper.py @@ -0,0 +1,15 @@ +import unittest +import ctree + + +@unittest.skipUnless(ctree.OCL_ENABLED, "OpenCL not enabled") +class TestCacheContexts(unittest.TestCase): + def test_simple_cache(self): + import pycl as cl + + from ctree.ocl import get_context_and_queue_from_devices + devices = cl.clGetDeviceIDs() + device = devices[-1] + results1 = get_context_and_queue_from_devices([device]) + results2 = get_context_and_queue_from_devices([device]) + self.assertEqual(results1, results2) diff --git a/test/test_ocl/test_types.py b/test/test_ocl/test_types.py deleted file mode 100644 index 3bbf4c9..0000000 --- a/test/test_ocl/test_types.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from ctree.c.nodes import SymbolRef -from ctree.ocl.types import * - - -class TestOclCodegen(unittest.TestCase): - def _check(self, tree, expected): - actual = str(tree) - self.assertEqual(actual, expected) - - def test_cl_device_id(self): - self._check(SymbolRef("foo", cl_device_id()), "cl_device_id foo") - - def test_cl_context(self): - self._check(SymbolRef("foo", cl_context()), "cl_context foo") - - def test_cl_command_queue(self): - self._check(SymbolRef("foo", cl_command_queue()), "cl_command_queue foo") - - def test_cl_program(self): - self._check(SymbolRef("foo", cl_program()), "cl_program foo") - - def test_cl_kernel(self): - self._check(SymbolRef("foo", cl_kernel()), "cl_kernel foo") - - def test_cl_mem(self): - self._check(SymbolRef("foo", cl_mem()), "cl_mem foo") - - def test_cl_mem_dot(self): - from ctree.dotgen import to_dot - - to_dot(SymbolRef("foo", cl_mem())) diff --git a/test/test_omp/test_macros.py b/test/test_omp/test_macros.py new file mode 100644 index 0000000..86ddea9 --- /dev/null +++ b/test/test_omp/test_macros.py @@ -0,0 +1,20 @@ +import unittest + +from ctree.omp.macros import * + + +class TestOmpMacros(unittest.TestCase): + def _check(self, actual, expected): + self.assertEqual(actual.codegen(), expected) + + def test_get_num_threads(self): + self._check(omp_get_num_threads(), "omp_get_num_threads()") + + def test_get_thread_num(self): + self._check(omp_get_thread_num(), "omp_get_thread_num()") + + def test_get_wtime(self): + self._check(omp_get_wtime(), "omp_get_wtime()") + + def test_include_omp_header(self): + self._check(IncludeOmpHeader(), "#include ") diff --git a/test/test_omp.py b/test/test_omp/test_nodes.py similarity index 51% rename from test/test_omp.py rename to test/test_omp/test_nodes.py index 2bac73f..e56b305 100644 --- a/test/test_omp.py +++ b/test/test_omp/test_nodes.py @@ -1,11 +1,12 @@ -import unittest +from textwrap import dedent +from ctypes import c_float from ctree.omp.nodes import * from ctree.omp.macros import * from ctree.c.nodes import * +from util import CtreeTest - -class TestOmpCodegen(unittest.TestCase): +class TestOmpCodegen(CtreeTest): def test_parallel(self): node = OmpParallel() self.assertEqual(str(node), "#pragma omp parallel") @@ -33,11 +34,45 @@ def test_ivdep(self): def test_no_semicolons(self): """There shouldn't be semicolons after Omp statementss.""" node = Block([OmpParallel(), Assign(SymbolRef("x"), Constant(3))]) - self.assertEqual(str(node), """{ - #pragma omp parallel - x = 3; -}""") + self.assertEqual(str(node), dedent("""\ + { + #pragma omp parallel + x = 3; + }""")) def test_get_wtime(self): node = omp_get_wtime() self.assertEqual(str(node), "omp_get_wtime()") + + def test_sections_1(self): + node = OmpParallelSections(sections=[ + OmpSection(body=[ + Assign(SymbolRef("i", c_float()), Constant(2)), + ]), + ]) + self._check_code(node, """\ + #pragma omp parallel sections + { + #pragma omp section + { + float i = 2; + } + }""") + + +class TestOmpMacros(CtreeTest): + def test_num_threads(self): + tree = omp_get_num_threads() + self._check_code(tree, "omp_get_num_threads()") + + def test_thread_num(self): + tree = omp_get_thread_num() + self._check_code(tree, "omp_get_thread_num()") + + def test_get_wtime(self): + tree = omp_get_wtime() + self._check_code(tree, "omp_get_wtime()") + + def test_include(self): + tree = IncludeOmpHeader() + self._check_code(tree, "#include ") diff --git a/test/test_parents.py b/test/test_parents.py deleted file mode 100644 index b658da1..0000000 --- a/test/test_parents.py +++ /dev/null @@ -1,29 +0,0 @@ -import unittest - -from ctree.c.nodes import * - - -class TestParentPointers(unittest.TestCase): - def test_parents_unop(self): - child = SymbolRef("foo") - minus_op = Sub(child) - self.assertEqual(child.parent, minus_op) - - def test_parents_binop(self): - child0, child1 = SymbolRef("foo"), Constant(12) - add_op = Add(child0, child1) - self.assertEqual(child0.parent, add_op) - self.assertEqual(child1.parent, add_op) - - def test_parents_augassign(self): - child0, child1 = SymbolRef("foo"), Constant(12) - add_op = AddAssign(child0, child1) - self.assertEqual(child0.parent, add_op) - self.assertEqual(child1.parent, add_op) - - def test_parents_while(self): - cond, body = SymbolRef("foo"), [Constant(12), SymbolRef("bar")] - node = While(cond, body) - self.assertEqual(cond.parent, node) - for child in node.body: - self.assertEqual(child.parent, node) diff --git a/test/test_pathrefs.py b/test/test_pathrefs.py index aeefba5..bb87ca5 100644 --- a/test/test_pathrefs.py +++ b/test/test_pathrefs.py @@ -1,35 +1,37 @@ import unittest +from ctypes import c_char_p +import ctree from ctree.nodes import * from ctree.c.nodes import * -from ctree.c.types import * -from ctree.jit import LazySpecializedFunction -class TestVerifyParentPointers(unittest.TestCase): +class TestPathRefs(unittest.TestCase): def test_self_ref(self): cfile = CFile("generated", []) tree = Project([cfile]) - stmt = Assign(SymbolRef("path", Ptr(Char())), \ + stmt = Assign(SymbolRef("path", c_char_p()), \ cfile.get_generated_path_ref()) cfile.body.append(stmt) proj = Project([cfile]) llvm_module = proj.codegen() # triggers path resolution - self.assertIsNone( proj.find(GeneratedPathRef) ) - self.assertIsNotNone( proj.find(String) ) + # self.assertIsNone( proj.find(GeneratedPathRef) ) + # self.assertIsNotNone( proj.find(String) ) + @unittest.skipUnless(ctree.OCL_ENABLED, "OpenCL not enabled") def test_other_ref(self): from ctree.ocl.nodes import OclFile + cfile = CFile("generated", []) oclfile = OclFile("kernel", []) tree = Project([cfile]) - stmt = Assign(SymbolRef("path", Ptr(Char())), \ + stmt = Assign(SymbolRef("path", c_char_p()), \ oclfile.get_generated_path_ref()) cfile.body.append(stmt) proj = Project([cfile]) llvm_module = proj.codegen() # triggers path resolution - - self.assertIsNone( proj.find(GeneratedPathRef) ) - self.assertIsNotNone( proj.find(String) ) + # + # self.assertIsNone( proj.find(GeneratedPathRef) ) + # self.assertIsNotNone( proj.find(String) ) diff --git a/test/test_precedence.py b/test/test_precedence.py index 0d7d1ec..c8e3f5d 100644 --- a/test/test_precedence.py +++ b/test/test_precedence.py @@ -1,4 +1,5 @@ import unittest +import ctypes as ct from ctree.c.nodes import * from ctree.precedence import * @@ -72,6 +73,16 @@ def test_postinc_unary(self): tree = PostInc(Sub(a)) self._check(tree, "(- a) ++") + def test_cast1(self): + a, b, c = self.args + tree = Add(Cast(ct.c_float(), a), b) + self._check(tree, "(float) a + b") + + def test_cast2(self): + a, b, c = self.args + tree = Cast(ct.c_float(), Add(a, b)) + self._check(tree, "(float) (a + b)") + class TestAssociativityPrecedence(unittest.TestCase): """ @@ -129,6 +140,12 @@ def test_bad_precedence_arg(self): with self.assertRaises(Exception): get_precedence(Constant(2.3)) + def test_bad_op(self): + a, b, c = self.args + tree = BinaryOp(a, b, c) + with self.assertRaises(Exception): + get_precedence(tree) + def test_bad_associativity_arg(self): with self.assertRaises(Exception): is_left_associative(Constant(2.3)) diff --git a/test/test_replacement.py b/test/test_replacement.py deleted file mode 100644 index d5c5929..0000000 --- a/test/test_replacement.py +++ /dev/null @@ -1,76 +0,0 @@ -import unittest -from textwrap import dedent - -from ctree.c.nodes import * - - -class TestAstListReplacement(unittest.TestCase): - def setUp(self): - self.front = SymbolRef("a") - self.mid = SymbolRef("b") - self.back = SymbolRef("c") - self.block = Block([ - self.front, - self.mid, - self.back, - ]) - - def _check(self, tree, expected_i): - actual = str(tree) - expected = dedent(expected_i) - self.assertEqual(actual, expected) - - def test_replace_list_front(self): - self.front.replace(SymbolRef("d")) - self._check(self.block, """\ - { - d; - b; - c; - }""") - - def test_replace_list_middle(self): - self.mid.replace(SymbolRef("d")) - self._check(self.block, """\ - { - a; - d; - c; - }""") - - def test_replace_list_back(self): - self.back.replace(SymbolRef("d")) - self._check(self.block, """\ - { - a; - b; - d; - }""") - - def test_bad_replace(self): - with self.assertRaises(Exception): - self.block.replace(SymbolRef("d")) - - -class TestAstFieldReplacement(unittest.TestCase): - def setUp(self): - self.lhs = SymbolRef("a") - self.rhs = SymbolRef("b") - self.binop = Add(self.lhs, self.rhs) - - def _check(self, tree, expected_i): - actual = str(tree) - expected = dedent(expected_i) - self.assertEqual(actual, expected) - - def test_replace_field_rhs(self): - self.rhs.replace(SymbolRef("x")) - self._check(self.binop, "a + x") - - def test_replace_field_lhs(self): - self.lhs.replace(SymbolRef("x")) - self._check(self.binop, "x + b") - - def test_bad_replace(self): - with self.assertRaises(Exception): - self.binop.replace(SymbolRef("d")) diff --git a/test/test_scopes.py b/test/test_scopes.py index 1874fdd..c725cba 100644 --- a/test/test_scopes.py +++ b/test/test_scopes.py @@ -26,7 +26,7 @@ def test_nested_00(self): a; 'b'; a; - }; + } 'b'; a; }""") diff --git a/test/test_specfuncs.py b/test/test_specfuncs.py index 36753b4..b611e70 100644 --- a/test/test_specfuncs.py +++ b/test/test_specfuncs.py @@ -1,26 +1,40 @@ import unittest from ctree.nodes import * -from ctree.c.nodes import * -from ctree.types import get_ctree_type - from ctree.jit import LazySpecializedFunction - +from ctree.jit import ConcreteSpecializedFunction +from ctree.frontend import dump from fixtures.sample_asts import * +import ctypes class TestTranslator(LazySpecializedFunction): def args_to_subconfig(self, args): - return {'arg_typesig': tuple(get_ctree_type(a) for a in args)} + return {'arg_typesig': tuple(type(get_ctype(a)) for a in args)} def transform(self, tree, program_config): arg_types = program_config[0]['arg_typesig'] - func_type = FuncType(arg_types[0], list(arg_types)) - tree.set_typesig(func_type) - tree = Project([CFile("generated", [tree])]), func_type.as_ctype() + tree.return_type = arg_types[0]() + for param, ty in zip(tree.params, arg_types): + param.type = ty() + return [CFile(tree.name, [tree])] + + def finalize(self, transform_result, program_config): + proj = Project(transform_result) + cfile = transform_result[0] + arg_types = program_config[0]['arg_typesig'] + + func_type = ctypes.CFUNCTYPE(arg_types[0], *arg_types) + return BasicFunction(cfile.name, proj, func_type) - return tree + +class BasicFunction(ConcreteSpecializedFunction): + def __init__(self, entry, tree, typesig): + self._c_function = self._compile(entry, tree, typesig) + + def __call__(self, *args, **kwargs): + return self._c_function(*args, **kwargs) class BadArgs(LazySpecializedFunction): @@ -30,55 +44,62 @@ def args_to_subconfig(self, args): class DefaultArgs(LazySpecializedFunction): def transform(self, tree, program_config): - return tree + return CFile("generated", [tree]) + + def finalize(self, transform_result, program_config): + proj = Project(transform_result) + ctype = self.original_tree.get_type().as_ctype() + return BasicFunction(self.original_tree.name, proj, ctype) class NoTransform(LazySpecializedFunction): def args_to_subconfig(self, args): - return {'arg_typesig': tuple(get_ctree_type(arg) for arg in args)} - - -class NoTuningSpace(NoTransform): - def transform(self, tree, program_config): - return tree + return {'arg_typesig': tuple(type(get_ctype(arg)) for arg in args)} +#@unittest.skip('Removed Support for AST injection') class TestSpecializers(unittest.TestCase): def test_identity_int(self): - c_identity = TestTranslator(identity_ast, "identity") + c_identity = TestTranslator(identity_ast, sub_dir='test_identity_int') self.assertEqual(c_identity(1), identity(1)) def test_identity_float(self): - c_identity = TestTranslator(identity_ast, "identity") - self.assertEqual(c_identity(1.2), identity(1.2)) - - def test_identity_intfloat(self): - c_identity = TestTranslator(identity_ast, "identity") - self.assertEqual(c_identity(1), identity(1)) + c_identity = TestTranslator(identity_ast, sub_dir='test_identity_float') self.assertEqual(c_identity(1.2), identity(1.2)) + def test_fib_int(self): - c_fib = TestTranslator(fib_ast, "fib") + c_fib = TestTranslator(fib_ast, sub_dir='test_fib_int') self.assertEqual(c_fib(1), fib(1)) + def test_fib_float(self): - c_fib = TestTranslator(fib_ast, "fib") + c_fib = TestTranslator(fib_ast, sub_dir='test_fib_float') self.assertEqual(c_fib(1.2), fib(1.2)) + def test_fib_intfloat(self): - c_fib = TestTranslator(fib_ast, "fib") + c_fib = TestTranslator(fib_ast, 'test_fib_intfloat') self.assertEqual(c_fib(1), fib(1)) self.assertEqual(c_fib(1.2), fib(1.2)) - + def test_gcd_int(self): - c_gcd = TestTranslator(gcd_ast, "gcd") + c_gcd = TestTranslator(gcd_ast, 'test_gcd_int') self.assertEqual(c_gcd(1, 2), gcd(1, 2)) - + def test_default_args_to_subconfig(self): - c_identity = DefaultArgs(identity_ast, "identity") + c_identity = DefaultArgs(identity_ast, 'test_default_args_to_subconfig') self.assertEqual(c_identity.args_to_subconfig([1, 2, 3]), {}) - + def test_no_transform(self): - c_identity = NoTransform(identity_ast, "identity") + c_identity = NoTransform(identity_ast, 'test_no_transform') with self.assertRaises(NotImplementedError): self.assertEqual(c_identity(1.2), identity(1.2)) + + def test_repeated(self): + c_fib = TestTranslator(fib_ast, 'test_repeated') + for i in range(20): + self.assertEqual(c_fib(1), fib(1)) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/test/test_symbols.py b/test/test_symbols.py index fa1fe2b..fd886db 100644 --- a/test/test_symbols.py +++ b/test/test_symbols.py @@ -1,27 +1,57 @@ import unittest +import ctypes as ct from ctree.c.nodes import * class TestSymbols(unittest.TestCase): + def _check(self, actual, expected): + self.assertEqual(actual.codegen(), expected) + def test_symbolref(self): ref = SymbolRef("foo") - assert str(ref) == "foo" + self._check(ref, "foo") def test_init_local(self): ref = SymbolRef("foo", _local=True) - assert str(ref) == "__local foo" + self._check(ref, "__local foo") def test_init_const(self): ref = SymbolRef("foo", _const=True) - assert str(ref) == "const foo" + self._check(ref, "const foo") def test_set_local(self): ref = SymbolRef("foo") ref.set_local() - assert str(ref) == "__local foo" + self._check(ref, "__local foo") def test_set_const(self): ref = SymbolRef("foo") ref.set_const() - assert str(ref) == "const foo" + self._check(ref, "const foo") + + def test_set_global(self): + ref = SymbolRef("foo") + ref.set_global() + self._check(ref, "__global foo") + + def test_unique(self): + ref1 = SymbolRef.unique("foo", ct.c_int()) + ref2 = SymbolRef.unique("foo", ct.c_int()) + self.assertNotEqual(ref1.codegen(), ref2.codegen()) + + def test_copy(self): + ref1 = SymbolRef("foo") + ref2 = ref1.copy() + self._check(ref1, ref2.codegen(())) + + def test_copy_without_declare(self): + ref1 = SymbolRef("foo", ct.c_int()) + ref2 = ref1.copy() + self._check(ref2, "foo") + + def test_copy_with_declare(self): + ref1 = SymbolRef("foo", ct.c_float()) + ref2 = ref1.copy(declare=True) + self._check(ref2, "float foo") + diff --git a/test/test_templates.py b/test/test_templates.py index ccfa8d0..2780c76 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -4,10 +4,9 @@ from ctree.templates.nodes import StringTemplate, FileTemplate from ctree.c.nodes import Constant, While -from ctree.dotgen import to_dot - import fixtures + class TestStringTemplates(unittest.TestCase): def _check(self, tree, expected): actual = tree.codegen() @@ -33,7 +32,7 @@ def test_dotgen(self): 'one': Constant(1), 'two': Constant(2), }) - dot = to_dot(tree) + dot = tree.to_dot() def test_indent_0(self): d = {'cond': Constant(1)} @@ -152,4 +151,4 @@ def test_file_template_dotgen(self): from ctree.c.nodes import String path = os.path.join(*(fixtures.__path__ + ["templates", "printf.tmpl.c"])) tree = FileTemplate(path, {'fmt': String('Hello, world!')}) - to_dot(tree) + tree.to_dot() diff --git a/test/test_transformations.py b/test/test_transformations.py new file mode 100644 index 0000000..bf28164 --- /dev/null +++ b/test/test_transformations.py @@ -0,0 +1,34 @@ +__author__ = 'nzhang-dev' + +import ast + +from ctree.transforms import DeclarationFiller +from ctree.transformations import PyBasicConversions +from ctree.frontend import * +from ctree.c.nodes import MultiNode + + +code = [ + "a = 1", + "a,b = 1,1", + "a = b = 1", + """a,b = 1,1 \na,b = b,a""" +] + +def fib(n): + a,b = 0, 1 + while n > 0: + n -= 1 + a, b = b, a+b + return a + +asts = [] +for c in code: + parsed = ast.parse(c) + asts.append(MultiNode(body = parsed.body)) + +asts.append(get_ast(fib).body[0]) + +processed = [ + DeclarationFiller().visit(PyBasicConversions().visit(a)) for a in asts +] diff --git a/test/test_transforms/__init__.py b/test/test_transforms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_transforms/test_constant_fold.py b/test/test_transforms/test_constant_fold.py new file mode 100644 index 0000000..d92da69 --- /dev/null +++ b/test/test_transforms/test_constant_fold.py @@ -0,0 +1,82 @@ +import unittest +import ctree.c.nodes as C +from ctree.transforms import ConstantFold + + +class TestConstantFold(unittest.TestCase): + def test_add_zero(self): + tree = C.Add(C.SymbolRef("a"), C.Constant(0)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.SymbolRef("a")) + + tree = C.Add(C.Constant(0), C.SymbolRef("a")) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.SymbolRef("a")) + + def test_add_constants(self): + tree = C.Add(C.Constant(20), C.Constant(10)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.Constant(30)) + + def test_sub_zero(self): + tree = C.Sub(C.SymbolRef("a"), C.Constant(0)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.SymbolRef("a")) + + tree = C.Sub(C.Constant(0), C.SymbolRef("a")) + tree = ConstantFold().visit(tree) + self.assertEqual(str(tree), str(C.Sub(C.SymbolRef("a")))) + + def test_no_folding(self): + trees = [ + C.Add(C.SymbolRef("a"), C.SymbolRef("b")), + C.Sub(C.SymbolRef("a"), C.SymbolRef("b")), + C.Mul(C.SymbolRef("a"), C.SymbolRef("b")), + C.Div(C.SymbolRef("a"), C.SymbolRef("b")), + ] + for tree in trees: + new_tree = ConstantFold().visit(tree) + self.assertEqual(tree, new_tree) + + def test_mul_constant(self): + tree = C.Mul(C.Constant(20), C.Constant(10)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.Constant(200)) + + def test_sub_constant(self): + tree = C.Sub(C.Constant(20), C.Constant(10)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.Constant(10)) + + def test_div_constant(self): + tree = C.Div(C.Constant(20), C.Constant(10)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.Constant(2)) + + def test_mul_by_0(self): + tree = C.Mul(C.Constant(0), C.SymbolRef("b")) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.Constant(0)) + + tree = C.Mul(C.SymbolRef("b"), C.Constant(0)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.Constant(0)) + + def test_mul_by_1(self): + tree = C.Mul(C.Constant(1), C.SymbolRef("b")) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.SymbolRef("b")) + + tree = C.Mul(C.SymbolRef("b"), C.Constant(1)) + tree = ConstantFold().visit(tree) + self.assertEqual(tree, C.SymbolRef("b")) + + def test_recursive_fold(self): + tree = C.Assign( + C.SymbolRef("c"), + C.Add(C.Add(C.Constant(2), C.Constant(-2)), + C.SymbolRef("b"))) + tree = ConstantFold().visit(tree) + self.assertEqual( + str(tree), + str(C.Assign(C.SymbolRef("c"), C.SymbolRef("b")))) diff --git a/test/test_transforms/test_declaration_filler.py b/test/test_transforms/test_declaration_filler.py new file mode 100644 index 0000000..04817bd --- /dev/null +++ b/test/test_transforms/test_declaration_filler.py @@ -0,0 +1,68 @@ +__author__ = 'nzhang-dev' + +import unittest + +from ctree.frontend import *; +from ctree.transformations import PyBasicConversions +from ctree.transforms import DeclarationFiller + + +def fib(n): + a, b = 0.0, 1.0 + k = "hello" + while n > 0: + a, b = b, a + b + n -= 1 + return a + + +class DeclarationTest(unittest.TestCase): + + def test_fib(self): + py_ast = get_ast(fib).body[0] + c_ast = PyBasicConversions().visit(py_ast) + filled_ast = DeclarationFiller().visit(c_ast) + print(filled_ast) + expected = """ +void fib(n) { + + double a = 0.0; + double b = 1.0; + + + char* k = "hello"; + + while (n > 0) { + + double ____temp__a = b; + double ____temp__b = a + b; + a = ____temp__a; + b = ____temp__b; + + n -= 1; + } + return a; +}""" + stripped_actual = str(filled_ast).replace(" ", "").replace("\n", "") + stripped_expected = expected.replace(" ", "").replace("\n", "") + self.assertEqual(stripped_actual, stripped_expected) + + def test_fmin(self): + def func(): + a = 3.0 + b = 4.0 + c = fmax(a + b, 0.0) + return c + py_ast = get_ast(func).body[0] + c_ast = PyBasicConversions().visit(py_ast) + filled_ast = DeclarationFiller().visit(c_ast) + expected = """ +void func() { + double a = 3.0; + double b = 4.0; + double c = fmax(a + b, 0.0); + return c; +}""" + stripped_actual = str(filled_ast).replace(" ", "").replace("\n", "") + stripped_expected = expected.replace(" ", "").replace("\n", "") + self.assertEqual(stripped_actual, stripped_expected) diff --git a/test/test_tuning.py b/test/test_tuning.py index c98f61d..0b38aea 100644 --- a/test/test_tuning.py +++ b/test/test_tuning.py @@ -1,26 +1,24 @@ import unittest -import os -import shutil from itertools import islice -class TestNullTuningDriver(unittest.TestCase): - def test_import(self): - import ctree.tune +# class TestNullTuningDriver(unittest.TestCase): +# def test_import(self): +# import ctree.tune - def test_null_driver_stream(self): - from ctree.tune import NullTuningDriver +# def test_null_driver_stream(self): +# from ctree.tune import NullTuningDriver - driver = NullTuningDriver() - for cfg in islice(driver.configs, 4): - self.assertDictEqual(cfg, {}) +# driver = NullTuningDriver() +# for cfg in islice(driver.configs, 4): +# self.assertDictEqual(cfg, {}) - def test_null_driver_report(self): - from ctree.tune import NullTuningDriver +# def test_null_driver_report(self): +# from ctree.tune import NullTuningDriver - driver = NullTuningDriver() - for cfg in islice(driver.configs, 4): - driver.report(time=0.4) +# driver = NullTuningDriver() +# for cfg in islice(driver.configs, 4): +# driver.report(time=0.4) try: @@ -122,3 +120,23 @@ def test_bruteforce_driver_2d_parabola(self): for config in islice(driver.configs, 10): self.assertEqual((config["x"], config["y"]), (3, 4)) + + def test_bruteforce_driver_other_params(self): + from ctree.tune import ( + BruteForceTuningDriver, + IntegerParameter, + BooleanParameter, + EnumParameter, + MinimizeTime, + ) + + params = [ + IntegerParameter("foo", 0, 10), + BooleanParameter("bar"), + EnumParameter("baz", ['monty', 'python', 'rocks']), + ] + driver = BruteForceTuningDriver(params, MinimizeTime()) + + nConfigs = 10*2*3 + configs = list(islice(driver.configs, nConfigs)) + self.assertEqual(len(configs), nConfigs) diff --git a/test/test_types.py b/test/test_types.py index e26c4ec..5a0f786 100644 --- a/test/test_types.py +++ b/test/test_types.py @@ -1,128 +1,114 @@ -import unittest - -from ctree.c.nodes import * -from ctree.c.types import * - - -class TestTypeProperties(unittest.TestCase): - def test_float_equality(self): - self.assertEqual(Float(), Float()) - - def test_unequality(self): - self.assertNotEqual(Float(), Int()) - - -class TestTypeFetcher(unittest.TestCase): - def _check(self, actual, expected): - self.assertEqual(actual, expected) - - def test_string_type(self): - s = String("foo") - self._check(s.get_type(), Ptr(Char())) - - def test_int_type(self): - n = Constant(123) - self._check(n.get_type(), Long()) - - def test_float_type(self): - n = Constant(123.4) - self._check(n.get_type(), Double()) - - def test_char_type(self): - n = Constant('b') - self._check(n.get_type(), Char()) - - def test_binop_add_intint(self): - a, b = Constant(1), Constant(2) - node = Add(a, b) - self._check(node.get_type(), Long()) - - def test_binop_add_floatfloat(self): - a, b = Constant(1.3), Constant(2.4) - node = Add(a, b) - self._check(node.get_type(), Double()) - - def test_binop_add_floatint(self): - a, b = Constant(1.3), Constant(2) - node = Add(a, b) - self._check(node.get_type(), Double()) - - def test_binop_add_intfloat(self): - a, b = Constant(1), Constant(2.3) - node = Add(a, b) - self._check(node.get_type(), Double()) - - def test_binop_add_charint(self): - a, b = Constant('b'), Constant(2) - node = Add(a, b) - self._check(node.get_type(), Long()) - - def test_binop_add_charfloat(self): - a, b = Constant('b'), Constant(2.3) - node = Add(a, b) - self._check(node.get_type(), Double()) - - def test_binop_compare_lessthan(self): - a, b = Constant('b'), Constant(2.3) - node = Lt(a, b) - self._check(node.get_type(), Int()) - - def test_binop_compare_comma(self): - a, b = Constant('b'), Constant(2.3) - node = Comma(a, b) - self._check(node.get_type(), Double()) - - def test_bad_constant(self): - class nothing: - pass - - a = Constant(nothing()) - with self.assertRaises(Exception): - self._check(a.get_type(), Int()) - - class Nothing: - pass - - def test_bad_type_coversion(self): - with self.assertRaises(Exception): - get_ctree_type(self.Nothing) - - def test_bad_obj_coversion(self): - with self.assertRaises(Exception): - get_ctree_type(self.Nothing()) - - -class BadType(CtreeType): - pass - - -class GoodType(CtreeType): - - def codegen(self): - pass - - def as_ctypes(self): - pass - - -class TestOverrideException(unittest.TestCase): - - def test_no_codegen(self): - with self.assertRaises(Exception): - BadType().codegen() - - def test_no_as_ctypes(self): - with self.assertRaises(Exception): - BadType().as_ctypes() - - def test_with_codegen(self): - try: - GoodType().codegen() - except Exception: - self.fail("codegen should not raise exception.") - - def test_with_as_ctypes(self): - try: - GoodType().as_ctypes() - except Exception: - self.fail("as_ctypes should not raise exception.") +import ctypes +import sys +from ctree.transforms.declaration_filler import DeclarationFiller + +from ctree.types import get_ctype, get_common_ctype +from util import CtreeTest +from ctree.c.nodes import SymbolRef, FunctionDecl, Assign, ArrayRef, \ + Constant, MultiNode, Dot + + +class TestTypeRecognizer(CtreeTest): + def test_int(self): + ty = get_ctype(123) + self.assertIsInstance(ty, ctypes.c_long) + + def test_float(self): + ty = get_ctype(456.7) + self.assertIsInstance(ty, ctypes.c_double) + + def test_char(self): + ty = get_ctype("c") + self.assertIsInstance(ty, ctypes.c_char) + + def test_none(self): + ty = get_ctype(None) + self.assertIsInstance(ty, type(None)) + + def test_bool(self): + ty = get_ctype(True) + self.assertIsInstance(ty, ctypes.c_bool) + + def test_string(self): + self.assertIsInstance(get_ctype("foo"), ctypes.c_char_p) + self.assertIsInstance(get_ctype(""), ctypes.c_char_p) + self.assertIsInstance(get_ctype("one two"), ctypes.c_char_p) + + def test_bad_type(self): + class Bad(object): pass + with self.assertRaises(ValueError): + ty = get_ctype(Bad()) + + +class TestTypeCodeGen(CtreeTest): + def test_int(self): + tree = SymbolRef("i", ctypes.c_int()) + self._check_code(tree, "int i") + + def test_long(self): + tree = SymbolRef("i", ctypes.c_long()) + if sys.maxsize > 2 ** 32: + self._check_code(tree, "long i") + else: + # int == long + self._check_code(tree, "int i") + + def test_float(self): + tree = SymbolRef("i", ctypes.c_double()) + self._check_code(tree, "double i") + + def test_char(self): + tree = SymbolRef("i", ctypes.c_char()) + self._check_code(tree, "char i") + + def test_none(self): + tree = SymbolRef("i", ctypes.c_void_p()) + self._check_code(tree, "void* i") + + def test_bool(self): + tree = SymbolRef("i", ctypes.c_bool()) + self._check_code(tree, "bool i") + + def test_string(self): + tree = SymbolRef("i", ctypes.c_char_p()) + self._check_code(tree, "char* i") + + def test_pointer(self): + tree = SymbolRef("i", ctypes.POINTER(ctypes.c_double)()) + self._check_code(tree, "double* i") + + def test_none(self): + tree = FunctionDecl(None, "foo", ()) + self._check_code(tree, "void foo()") + + def test_bad_type(self): + class Bad(object): pass + with self.assertRaises(ValueError): + SymbolRef("i", Bad()).codegen() + +class TestTypeCoercion(CtreeTest): + def test_coercion(self): + types = (ctypes.c_long, ctypes.c_double, ctypes.c_int) + self.assertEqual(get_common_ctype(types), ctypes.c_double) + + +class TestBinaryOpTypeInference(CtreeTest): + def test_array_ref(self): + tree = MultiNode([ + SymbolRef("foo", ctypes.POINTER(ctypes.c_double)()), + Assign(SymbolRef("____temp__x"), ArrayRef(SymbolRef("foo"), Constant(0))) + ]) + DeclarationFiller().visit(tree) + self._check_code(tree, "\ndouble* foo;\n" + "double ____temp__x = foo[0];\n") + + def test_dot(self): + op = SymbolRef("op") + setattr(op, "get_type", lambda: ctypes.c_char()) + + foo = SymbolRef("foo") + setattr(foo, "get_type", lambda: ctypes.c_double()) + + tree = Assign(SymbolRef("x"), Dot(foo, op)) + DeclarationFiller().visit(tree) + self._check_code(tree, "char x = foo . op") diff --git a/test/test_unops.py b/test/test_unops.py index c493479..af0cb4c 100644 --- a/test/test_unops.py +++ b/test/test_unops.py @@ -1,6 +1,8 @@ import unittest from ctree.c.nodes import * +from ctree.transformations import PyBasicConversions +import ast class TestUnaryOps(unittest.TestCase): @@ -40,3 +42,35 @@ def test_postinc(self): def test_postdec(self): self._check(PostDec, "foo --") + + def test_sizeof(self): + self._check(SizeOf, "sizeof foo") + +class TestPyBasicConversionsUnaryOps(unittest.TestCase): + def _check(self, op, expected_string): + self.assertEqual(str(op), expected_string) + + def test_plus(self): + op = ast.parse("+ foo") + op = PyBasicConversions().visit(op).find(UnaryOp) + self._check(op, "+ foo") + + def test_minus(self): + op = ast.parse("- foo") + op = PyBasicConversions().visit(op).find(UnaryOp) + self._check(op, "- foo") + + def test_bitnot(self): + op = ast.parse("~ foo") + op = PyBasicConversions().visit(op).find(UnaryOp) + self._check(op, "~ foo") + + def test_not(self): + op = ast.parse("not foo") + op = PyBasicConversions().visit(op).find(UnaryOp) + self._check(op, "! foo") + + def test_CUnaryOp(self): + op = Not(SymbolRef("foo")) + op = PyBasicConversions().visit(op).find(UnaryOp) + self._check(str(op), "! foo") diff --git a/test/test_util.py b/test/test_util.py index 7f3906e..556721d 100644 --- a/test/test_util.py +++ b/test/test_util.py @@ -1,6 +1,9 @@ import unittest +import math -from ctree.util import * +from ctree.util import truncate, product, strides, flatten +from ctree.util import lower_case_underscore_to_camel_case +from ctree.util import singleton class TestTruncate(unittest.TestCase): @@ -15,6 +18,41 @@ def test_trunctate(self): self.assertNotEqual(truncate(text), text) +@singleton +class Single(object): + def id(self): + return self.__hash__() + + +class TestUtil(unittest.TestCase): + def _check(self, actual, expected): + self.assertEqual(actual, expected) + + def test_singleton(self): + s1 = Single + s2 = Single + + self.assertRaises(TypeError, Single) + + self.assertEqual(s1.id, s2.id) + + def test_truncate(self): + s = "foo" + s_truncated = truncate(s) + self.assertEqual(s, s_truncated) + + many_lines = "\n".join(map(lambda x: "line %d" % x, [l for l in range(500)])) + less_lines = truncate(many_lines) + + self.assertTrue("lines suppressed" in less_lines) + + def test_lower_case_to_camel_case(self): + s = "dog_cat" + cc = lower_case_underscore_to_camel_case(s) + + self.assertEqual(cc,"DogCat") + + class TestLowerCaseUnderscoreToCamelCase(unittest.TestCase): def test_simple(self): text = 'this_is_a_name' @@ -22,3 +60,14 @@ def test_simple(self): lower_case_underscore_to_camel_case(text), 'ThisIsAName' ) + +class TestMath(unittest.TestCase): + def test_product(self): + self.assertEqual(product(range(1, 11)), math.factorial(10)) + + def test_strides(self): + self.assertEqual(strides((3, 3, 3)), [9, 3, 1]) + + def test_flatten(self): + l = [1, 2, 3, [4, 5, [6, 7], [8, 9]]] + self.assertEqual(list(flatten(l)), list(range(1, 10))) diff --git a/test/test_visitors.py b/test/test_visitors.py index e12b038..651180e 100644 --- a/test/test_visitors.py +++ b/test/test_visitors.py @@ -1,5 +1,3 @@ -import unittest - from ctree.visitors import NodeVisitor diff --git a/test/test_xforms.py b/test/test_xforms.py index ecf1205..627a7f8 100644 --- a/test/test_xforms.py +++ b/test/test_xforms.py @@ -1,68 +1,11 @@ -import ast -import sys import unittest from fixtures.sample_asts import * from ctree.transformations import * from ctree.c.nodes import * -from ctree.c.types import * from ctree.frontend import get_ast -class TestSetTypeSig(unittest.TestCase): - def _check(self, func_type, tree): - if isinstance(tree, FunctionDecl): - self.assertEqual(tree.return_type, func_type.return_type) - for param, expected_type in zip(tree.params, func_type.arg_types): - self.assertEqual(param.type, expected_type) - elif isinstance(tree, ast.Module): - self._check(func_type, tree.body[0]) - else: - self.fail("Can't check param setting on %s object." % tree) - - def test_no_args(self): - func_type = FuncType(Long()) - get_two_ast.set_typesig(func_type) - self._check(func_type, get_two_ast) - - def test_one_arg(self): - func_type = FuncType(Long(), [Long()]) - fib_ast.set_typesig(func_type) - self._check(func_type, fib_ast) - - def test_two_args(self): - func_type = FuncType(Long(), [Long(), Long()]) - gcd_ast.set_typesig(func_type) - self._check(func_type, gcd_ast) - - def test_mixed_args(self): - func_type = FuncType(Long(), [Double(), Long(), Long()]) - choose_ast.set_typesig(func_type) - self._check(func_type, choose_ast) - - -class TestFixUpParentPointers(unittest.TestCase): - def _check(self, root): - from ctree.analyses import VerifyParentPointers - - VerifyParentPointers().visit(root) - - def test_identity(self): - identity_ast.find(SymbolRef, name="x").parent = None - tree = FixUpParentPointers().visit(identity_ast) - self._check(tree) - - def test_fib(self): - fib_ast.find(Constant, value=2).parent = None - tree = FixUpParentPointers().visit(fib_ast) - self._check(tree) - - def test_gcd(self): - gcd_ast.find(Return).parent = None - tree = FixUpParentPointers().visit(gcd_ast) - self._check(tree) - - class TestCtxScrubber(unittest.TestCase): def _check(self, tree): for node in ast.walk(tree): @@ -110,11 +53,11 @@ def test_subtree_docstrings(self): ])) self._check(tree) - class TestBasicConversions(unittest.TestCase): - def _check(self, py_ast, expected_c_ast): - actual_c_ast = PyBasicConversions().visit(py_ast) - self.assertEqual(str(actual_c_ast), str(expected_c_ast)) + def _check(self, py_ast, expected_c_ast, names_dict ={}, constants_dict={}): + actual_c_ast = PyBasicConversions(names_dict, constants_dict).visit(py_ast) + self.assertEqual(str(actual_c_ast).strip('\n;'), str(expected_c_ast).strip('\n;')) + def test_num_float(self): py_ast = ast.Num(123.4) @@ -137,8 +80,29 @@ def test_name(self): self._check(py_ast, c_ast) def test_binop(self): - py_ast = ast.BinOp(ast.Num(1), ast.Add(), ast.Num(2)) - c_ast = Add(Constant(1), Constant(2)) + for py_op, c_op in ( + (ast.Add, Add), + (ast.Sub, Sub), + (ast.BitXor, BitXor), + (ast.BitAnd, BitAnd), + (ast.BitOr, BitOr) + ): + py_ast = ast.BinOp(ast.Num(1), py_op(), ast.Num(2)) + c_ast = c_op(Constant(1), Constant(2)) + self._check(py_ast, c_ast) + + def test_boolop(self): + def fn(): + 1 or 2 or 3 + py_ast = get_ast(fn).body[0].body[0] + c_ast = Or(Or(Constant(1), Constant(2)), Constant(3)) + self._check(py_ast, c_ast) + + def test_compare(self): + def fn(): + 0 < a < 3 + py_ast = get_ast(fn).body[0].body[0] + c_ast = And(Gt(0, SymbolRef("a")), Lt(SymbolRef("a"), Constant(3))) self._check(py_ast, c_ast) def test_return(self): @@ -172,6 +136,7 @@ def test_arg(self): c_ast = SymbolRef("foo") self._check(py_ast, c_ast) + @unittest.skipIf(sys.version_info.major == 3, "Different ast.Call class in Python 3") def test_for_1_arg(self): stop = ast.Num(10) py_ast = ast.For(ast.Name("i", ast.Load()), @@ -179,7 +144,7 @@ def test_for_1_arg(self): [ast.Name("foo", ast.Load())], [], ) - i = SymbolRef("i", Long()) + i = SymbolRef("i", c_long()) c_ast = For( Assign(i, Constant(0)), Lt(i.copy(), Constant(10)), @@ -188,6 +153,7 @@ def test_for_1_arg(self): ) self._check(py_ast, c_ast) + @unittest.skipIf(sys.version_info.major == 3, "Different ast.Call class in Python 3") def test_for_2_args(self): start = ast.Num(2) stop = ast.Num(10) @@ -196,7 +162,7 @@ def test_for_2_args(self): [ast.Name("foo", ast.Load())], [], ) - i = SymbolRef("i", Long()) + i = SymbolRef("i", c_long()) c_ast = For( Assign(i, Constant(2)), Lt(i.copy(), Constant(10)), @@ -205,6 +171,7 @@ def test_for_2_args(self): ) self._check(py_ast, c_ast) + @unittest.skipIf(sys.version_info.major == 3, "Different ast.Call class in Python 3") def test_for_3_args(self): start = ast.Num(2) stop = ast.Num(10) @@ -214,7 +181,7 @@ def test_for_3_args(self): [ast.Name("foo", ast.Load())], [], ) - i = SymbolRef("i", Long()) + i = SymbolRef("i", c_long()) c_ast = For( Assign(i, Constant(2)), Lt(i.copy(), Constant(10)), @@ -223,6 +190,7 @@ def test_for_3_args(self): ) self._check(py_ast, c_ast) + @unittest.skipIf(sys.version_info.major == 3, "Different ast.Call class in Python 3") def test_for_0_args(self): py_ast = ast.For(ast.Name("i", ast.Load()), ast.Call(ast.Name("range", ast.Load()), [], [], None, None), @@ -232,6 +200,7 @@ def test_for_0_args(self): with self.assertRaises(Exception): self._check(py_ast, None) + @unittest.skipIf(sys.version_info.major == 3, "Different ast.Call class in Python 3") def test_for_4_args(self): py_ast = ast.For(ast.Name("i", ast.Load()), ast.Call(ast.Name("range", ast.Load()), @@ -242,6 +211,7 @@ def test_for_4_args(self): with self.assertRaises(Exception): self._check(py_ast, None) + @unittest.skipIf(sys.version_info.major == 3, "Different ast.Call class in Python 3") def test_for_expr_args(self): start = ast.BinOp(ast.Num(2), ast.Add(), ast.Num(3)) stop = ast.BinOp(ast.Num(4), ast.Mult(), ast.Num(10)) @@ -251,7 +221,7 @@ def test_for_expr_args(self): [ast.Name("foo", ast.Load())], [], ) - i = SymbolRef("i", Long()) + i = SymbolRef("i", c_long()) c_ast = For( Assign(i, Add(Constant(2), Constant(3))), Lt(i.copy(), Mul(Constant(4), Constant(10))), @@ -282,3 +252,65 @@ def test_DivAssign(self): ast.Div(), ast.Num(3)) c_ast = DivAssign(SymbolRef('i'), Constant(3)) self._check(py_ast, c_ast) + + def test_AugAssign(self): + + for py_op, c_op in ( + ( + (ast.Div, DivAssign), + (ast.Add, AddAssign), + (ast.Mult, MulAssign), + (ast.BitOr, BitOrAssign), + (ast.BitAnd, BitAndAssign), + (ast.BitXor, BitXorAssign), + (ast.LShift, BitShLAssign), + (ast.RShift, BitShRAssign) + ) + ): + py_ast = ast.AugAssign(ast.Name('i', ast.Load()), + py_op(), ast.Num(3)) + c_ast = c_op(SymbolRef('i'), Constant(3)) + self._check(py_ast, c_ast) + + def test_Assign(self): + py_ast = ast.Assign([ast.Name('i', ast.Load())], + ast.Num(3)) + c_ast = Assign(SymbolRef('i'), Constant(3)) + self._check(py_ast, c_ast) + + def test_namesDict(self): + py_ast = ast.Name('i',ast.Load()) + c_ast = SymbolRef('d') + self._check(py_ast,c_ast,names_dict={'i':'d'}) + + def test_constantsDict(self): + py_ast = ast.Name('i',ast.Load()) + c_ast = Constant(234) + self._check(py_ast,c_ast,constants_dict={'i':234}) + + def test_Subscript(self): + py_ast = ast.Subscript(value=ast.Name('i',ast.Load()), + slice=ast.Index(value=ast.Num(n=1), ctx=ast.Load())) + c_ast = ArrayRef(SymbolRef('i'),Constant(1)) + self._check(py_ast,c_ast) + + def test_Range_ValueError(self): + py_ast = ast.For(target=ast.Name(id='i', ctx=ast.Store()), iter=ast.Call(func=ast.Name(id='range', ctx=ast.Load()), args=[ + ast.Num(n=1), + ast.Num(n=0), + ast.Num(n=0), + ], keywords=[], starargs=None, kwargs=None), body=[ + Pass(), + ], orelse=[]) + with self.assertRaises(ValueError): + PyBasicConversions().visit(py_ast) + + def test_Range_NoOp(self): + py_ast = ast.For(target=ast.Name(id='i', ctx=ast.Store()), iter=ast.Call(func=ast.Name(id='range', ctx=ast.Load()), args=[ + ast.Num(n=1), + ast.Num(n=1), + ast.Num(n=3), + ], keywords=[], starargs=None, kwargs=None), body=[ + Pass(), + ], orelse=[]) + self.assertEqual(PyBasicConversions().visit(py_ast), None) diff --git a/test/util.py b/test/util.py index 265bb5c..b3ce39b 100644 --- a/test/util.py +++ b/test/util.py @@ -1,3 +1,10 @@ +import unittest +import difflib +import textwrap + +from ctree.util import highlight + + class PreventImport(object): """ Context manager that overrides the builtin __import__ method. @@ -31,3 +38,25 @@ def __enter__(self): def __exit__(self, excp, traceback, value): __builtins__['__import__'] = self.__import__ + + +class CtreeTest(unittest.TestCase): + def _check_code(self, actual="", expected=""): + if not isinstance(actual, str): + actual = textwrap.dedent( str(actual) ) + if not isinstance(expected, str): + expected = textwrap.dedent( str(expected) ) + + actual = textwrap.dedent(str(actual)) + expected = textwrap.dedent(str(expected)) + + if actual != expected: + actual_display = (actual + ("\n" if actual[-1] != "\n" else "")).splitlines(True) + expected_display = expected.splitlines(True) + diff_gen = difflib.unified_diff( + actual_display, expected_display, + "", "") + diff = "".join(diff_gen) + print(highlight(diff, language='diff')) + + self.assertEqual(actual, expected)