Add Python language support.

sed-inf-u-szeged · Mar 6, 2018 · fc1d859 · fc1d859
1 parent c6ae9e3
commit fc1d859
Show file tree

Hide file tree

Showing 340 changed files with 88,127 additions and 4,340 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -10,3 +10,9 @@
 [submodule "java/lib/langtools"]
 	path = java/lib/langtools
 	url = https://github.com/sed-inf-u-szeged/langtools.git
+[submodule "3rdparty/python2"]
+	path = 3rdparty/python2
+	url = ../cpython
+[submodule "3rdparty/python3"]
+	path = 3rdparty/python3
+	url = ../cpython
diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt
@@ -88,9 +88,9 @@ endif ()
 
 if (CMAKE_SYSTEM_NAME STREQUAL Linux)
   ExternalProject_Add( xerces-c
-      SVN_REPOSITORY https://svn.apache.org/repos/asf/xerces/c/tags/Xerces-C_3_1_4
-      BUILD_IN_SOURCE 1
-      UPDATE_COMMAND ""
+      URL http://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.4.tar.gz
+	  URL_HASH MD5=21bb097b711a513275379b59757cba4c 
+	  BUILD_IN_SOURCE 1
       CONFIGURE_COMMAND ./reconf
                 COMMAND ./configure --disable-shared -disable-network --disable-transcoder-icu --prefix=${COLUMBUS_3RDPARTY_INSTALL_DIR}
       BUILD_COMMAND make -j8 all
@@ -111,16 +111,15 @@ elseif (CMAKE_SYSTEM_NAME STREQUAL Windows)
   endif ()
 
   ExternalProject_Add( xerces-c
-      SVN_REPOSITORY https://svn.apache.org/repos/asf/xerces/c/tags/Xerces-C_3_1_4
+	  URL http://archive.apache.org/dist/xerces/c/3/sources/xerces-c-3.1.4.zip
+	  URL_HASH MD5=6fcd8ec268f6bfe11d8ce2cd7d25a185
       BUILD_IN_SOURCE 1
-      UPDATE_COMMAND ""
       CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo "No configuration is required."
       BUILD_COMMAND msbuild projects/Win32/VC14/xerces-all/xerces-all.sln /m /t:XercesLib "/p:Configuration=Static $<CONFIG>" /p:Platform=${VS_PLATFORM} /p:PlatformToolset=${CMAKE_VS_PLATFORM_TOOLSET}
       INSTALL_COMMAND ${CMAKE_COMMAND} "-DCOPY_SOURCE=${CMAKE_CURRENT_BINARY_DIR}/xerces-c-prefix/src/xerces-c/src/xercesc/" "-DCOPY_TARGET=${COLUMBUS_3RDPARTY_INSTALL_DIR}/include/xercesc" -DCOPY_PATTERN=*.hpp -P ${CMAKE_CURRENT_SOURCE_DIR}/CMake/SearchAndCopyFiles.cmake
               COMMAND ${CMAKE_COMMAND} "-DCOPY_SOURCE=${CMAKE_CURRENT_BINARY_DIR}/xerces-c-prefix/src/xerces-c/src/xercesc/" "-DCOPY_TARGET=${COLUMBUS_3RDPARTY_INSTALL_DIR}/include/xercesc" -DCOPY_PATTERN=*.c -P ${CMAKE_CURRENT_SOURCE_DIR}/CMake/SearchAndCopyFiles.cmake
               COMMAND ${CMAKE_COMMAND} -E copy "Build/${XERCESC_BUILD_OUTPUT_DIR}/VC14/Static $<CONFIG>/xerces-c_static_3$<$<CONFIG:Debug>:d>.lib" ${COLUMBUS_3RDPARTY_INSTALL_DIR}/lib/xerces-c.lib
       LOG_DOWNLOAD 1
-      LOG_UPDATE 1
       LOG_CONFIGURE 1
       LOG_BUILD 1
       LOG_INSTALL 1
@@ -151,3 +150,69 @@ add_custom_target (
 )
 
 set_target_properties (PMD PROPERTIES FOLDER "ExternalProjectTargets/pmd")
+
+########################### python27 #########################
+if (CMAKE_SYSTEM_NAME STREQUAL Linux)
+
+  ExternalProject_Add( python2.7
+      DEPENDS zlib
+      SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python2
+      BUILD_IN_SOURCE 1
+      CONFIGURE_COMMAND ./configure --enable-optimizations --disable-shared --prefix=${COLUMBUS_3RDPARTY_INSTALL_DIR}
+      BUILD_COMMAND make -j8 altbininstall inclinstall
+      INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "No install step is required as the build step installs the library."
+      LOG_CONFIGURE 1
+      LOG_BUILD 1
+      LOG_INSTALL 1
+  )
+elseif (CMAKE_SYSTEM_NAME STREQUAL Windows)
+  if (COLUMBUS_64BIT)
+    set (PYTHON_BUILD_OUTPUT_DIR amd64)
+  endif ()
+
+  ExternalProject_Add( python2.7
+      DEPENDS zlib
+      SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python2
+      BUILD_IN_SOURCE 1
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND msbuild PCbuild/pythoncore.vcxproj /m /t:Build /p:Configuration=$<CONFIG> /p:Platform=${VS_PLATFORM} /p:PlatformToolset=${CMAKE_VS_PLATFORM_TOOLSET}
+      INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory Include ${COLUMBUS_3RDPARTY_INSTALL_DIR}/include/python2.7
+              COMMAND ${CMAKE_COMMAND} -E copy "PC/pyconfig.h" ${COLUMBUS_3RDPARTY_INSTALL_DIR}/include/python2.7/pyconfig.h
+              COMMAND ${CMAKE_COMMAND} -E copy "PCbuild/${PYTHON_BUILD_OUTPUT_DIR}/python27$<$<CONFIG:Debug>:_d>.lib" ${COLUMBUS_3RDPARTY_INSTALL_DIR}/lib/python2.7.lib
+      LOG_CONFIGURE 1
+      LOG_BUILD 1
+      LOG_INSTALL 1
+  )
+
+endif ()
+
+########################### python36 #########################
+if (CMAKE_SYSTEM_NAME STREQUAL Linux)
+
+  ExternalProject_Add( python3.6
+      DEPENDS zlib
+      SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python3
+      BUILD_IN_SOURCE 1
+      CONFIGURE_COMMAND ./configure --enable-optimizations --disable-shared --prefix=${COLUMBUS_3RDPARTY_INSTALL_DIR} --without-pymalloc
+      BUILD_COMMAND make -j8 altbininstall inclinstall
+      INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "No install step is required as the build step installs the library."
+      LOG_CONFIGURE 1
+      LOG_BUILD 1
+      LOG_INSTALL 1
+  )
+elseif (CMAKE_SYSTEM_NAME STREQUAL Windows)
+
+  ExternalProject_Add( python3.6
+      DEPENDS zlib
+      SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/python3
+      BUILD_IN_SOURCE 1
+      CONFIGURE_COMMAND ""
+      BUILD_COMMAND msbuild PCbuild/pythoncore.vcxproj /m /t:Build /p:Configuration=$<CONFIG> /p:Platform=${VS_PLATFORM} /p:PlatformToolset=${CMAKE_VS_PLATFORM_TOOLSET}
+      INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_directory Include ${COLUMBUS_3RDPARTY_INSTALL_DIR}/include/python3.6
+              COMMAND ${CMAKE_COMMAND} -E copy "PC/pyconfig.h" ${COLUMBUS_3RDPARTY_INSTALL_DIR}/include/python3.6/pyconfig.h
+              COMMAND ${CMAKE_COMMAND} -E copy "PCbuild/${PYTHON_PLATFORM_NAME}/python36$<$<CONFIG:Debug>:_d>.lib" ${COLUMBUS_3RDPARTY_INSTALL_DIR}/lib/python3.6.lib
+      LOG_CONFIGURE 1
+      LOG_BUILD 1
+      LOG_INSTALL 1
+  )
+endif ()
diff --git a/3rdparty/python/pylint/bin/pylint b/3rdparty/python/pylint/bin/pylint
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+import sys
+
+if __name__ == '__main__':
+    from pylint.lint import Run
+    Run(sys.argv[1:], exit=False)
+    sys.exit(0)
diff --git a/3rdparty/python2 b/3rdparty/python2
diff --git a/3rdparty/python3 b/3rdparty/python3
diff --git a/CMake/PlatformOptions.cmake b/CMake/PlatformOptions.cmake
@@ -10,10 +10,12 @@ if (COLUMBUS_64BIT)
   set (COLUMBUS_PLATFORM_NAME x64)
   set (VS_PLATFORM x64)
   set (VS_PLATFORM_DIR x64/)
+  set (PYTHON_PLATFORM_NAME amd64)
 else ()
   set (COLUMBUS_PLATFORM_NAME x86)
   set (VS_PLATFORM WIN32)
   set (VS_PLATFORM_DIR )
+  set (PYTHON_PLATFORM_NAME win32)
 endif ()
 
 if (CMAKE_SYSTEM_NAME STREQUAL Windows)

diff --git a/CMake/UtilityFunctions.cmake b/CMake/UtilityFunctions.cmake
@@ -20,6 +20,9 @@ function (set_schema_language_compiler_settings TARGET SCHEMA)
   if (SCHEMA STREQUAL "java")
     target_compile_definitions(${TARGET} PUBLIC SCHEMA_JAVA)
     target_link_libraries(${TARGET} java)
+  elseif (SCHEMA STREQUAL "python")
+    target_compile_definitions(${TARGET} PUBLIC SCHEMA_PYTHON)
+    target_link_libraries(${TARGET} python)
   else ()
     message (FATAL_ERROR "Unknown schema language: ${SCHEMA}")
   endif ()

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -8,7 +8,7 @@ if (NOT CMAKE_BUILD_TYPE )
   set (CMAKE_BUILD_TYPE Release CACHE STRING "Build Type: Release or Debug" FORCE)
 endif()
 
-project (OSA VERSION 1.0.0)
+project (OSA VERSION 2.0.0)
 
 set (EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/bin)
 set (CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${EXECUTABLE_OUTPUT_PATH})
@@ -38,8 +38,12 @@ add_subdirectory (cl/JANFilter)
 add_subdirectory (cl/JANLink)
 add_subdirectory (cl/LIM2Metrics)
 add_subdirectory (cl/MetricHunter)
+add_subdirectory (cl/PAN)
+add_subdirectory (cl/PAN2Lim)
 add_subdirectory (cl/PMD2Graph)
+add_subdirectory (cl/Pylint2Graph)
 add_subdirectory (cl/OpenStaticAnalyzerJava)
+add_subdirectory (cl/OpenStaticAnalyzerPython)
 
 add_subdirectory (wrapper/AbstractWrapperLib)
 add_subdirectory (wrapper/AnalyzerWrapperConfig)
@@ -61,6 +65,7 @@ add_subdirectory (lib/java)
 add_subdirectory (lib/lim)
 add_subdirectory (lib/lim2graph)
 add_subdirectory (lib/limmetrics)
+add_subdirectory (lib/python)
 add_subdirectory (lib/rul)
 add_subdirectory (lib/strtable)
 add_subdirectory (lib/threadpool)

diff --git a/OpenStaticAnalyzer/CMakeLists.txt b/OpenStaticAnalyzer/CMakeLists.txt
@@ -13,6 +13,7 @@ endif ()
 add_custom_target (
     ${OSA_TARGET_NAME}
     DEPENDS OSA-Java
+    DEPENDS OSA-Python
     COMMAND ${CMAKE_COMMAND} -E ${COMPRESS_COMMAND}
 )
 
@@ -49,3 +50,4 @@ function (copy_licenses)
 endfunction ()
 
 add_subdirectory (java)
+add_subdirectory (python)
diff --git a/OpenStaticAnalyzer/doc/readme.txt b/OpenStaticAnalyzer/doc/readme.txt
@@ -1,6 +1,6 @@
 OpenStaticAnalyzer is a source code analyzer tool, which can perform deep
 static analysis of the source code of complex software systems implemented in
-Java programming languages.
+Java or Python programming languages.
 
 The source code of a program is usually its only up-to-date documentation. At
 the same time, the source code is the exquisite bearer of knowledge, business
@@ -13,10 +13,13 @@ and is merely considered as a tool.
 Product characteristics
 
 The most important product characteristics of OpenStaticAnalyzer are the following:
-* Support Java 8
+* Support for Java 8, Python 2.7.x and 3.6.x
 * Platform-independent command line tools
 * Transparent integration into build processes
 * Powerful filter management
+* Coding issue detection
+  - Metric threshold violations
+  - Integration of popular free tools (PMD, FindBugs, Pylint)
 * Clone detection (copy-pasted source code fragments) extended with clone
   tracking and "clone smells"
   - Syntax-based, so-called Type-2 clones
@@ -35,4 +38,4 @@ By continuous static analysis, the software developers can:
   operational risks can be decreased, increasing the company's reputation.
 
 For more information about OpenStaticAnalyzer please read the files UsersGuide.html,
-which can be found in the Java subdirectory of the package.
+which can be found in the Java and Python subdirectories of the package.
diff --git a/OpenStaticAnalyzer/java/CMakeLists.txt b/OpenStaticAnalyzer/java/CMakeLists.txt
@@ -46,8 +46,8 @@ add_custom_generated_copy_dependency_to_tools_dir (GraphMerge${EXE})
 add_custom_generated_copy_dependency_to_tools_dir (JAN2ChangePath${EXE})
 add_custom_generated_copy_dependency_to_tools_dir (JAN2Lim${EXE})
 add_custom_generated_copy_dependency_to_tools_dir (JANFilter${EXE})
-add_custom_generated_copy_dependency_to_tools_dir (LIM2Metrics_java${EXE} LIM2Metrics_java LIM2Metrics${EXE})
-add_custom_generated_copy_dependency_to_tools_dir (MET.rul LIM2Metrics_java)
+add_custom_generated_copy_dependency_to_tools_dir (LIM2Metrics${EXE})
+add_custom_generated_copy_dependency_to_tools_dir (MET.rul LIM2Metrics)
 add_custom_generated_copy_dependency_to_tools_dir (MetricHunter_java${EXE} MetricHunter_java MetricHunter${EXE})
 add_custom_generated_copy_dependency_to_tools_dir (MetricHunter_JAVA.threshold MetricHunter_java_copy_MetricHunter_JAVA.threshold MetricHunter.threshold)
 add_custom_generated_copy_dependency_to_tools_dir (PMD2Graph${EXE})
@@ -137,16 +137,16 @@ set (MAVEN_PLUGIN_DIR ${CMAKE_SOURCE_DIR}/java/columbus-toolchain-maven-plugin)
 
 add_custom_create_file_target (
     ${OSA_TARGET_NAME}
-    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-8.2.jar
+    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-2.0.jar
     ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-Maven-plugin-mojo-executer-2.2.1.jar
     ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-Maven-plugin-mojo-executer-2.2.1.pom
     ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-Maven-plugin-mojo-executer-3.0.jar
     ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-Maven-plugin-mojo-executer-3.0.pom
     ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-Maven-plugin-mojo-executer-3.1.jar
     ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-Maven-plugin-mojo-executer-3.1.pom
-    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-8.2-V2.pom
-    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-8.2-V3.pom
-    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-8.2-V31.pom
+    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-2.0-V2.pom
+    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-2.0-V3.pom
+    ${WRAPPER_TOOLS_DIR}/OpenStaticAnalyzer-maven-plugin-2.0-V31.pom
     ${OSA_PACKAGE_DIR}/${PACKAGE_LANG}/installMavenWrapper${SCRIPT_EXT}
     COMMAND mvn${MVN_POSTFIX} install -f ${MAVEN_PLUGIN_DIR}/ext/mojoExecutorModule/v3/pom.xml > ${CMAKE_CURRENT_BINARY_DIR}/v3install.log
     COMMAND mvn${MVN_POSTFIX} install -f ${MAVEN_PLUGIN_DIR}/ext/mojoExecutorModule/v31/pom.xml > ${CMAKE_CURRENT_BINARY_DIR}/v31install.log

diff --git a/OpenStaticAnalyzer/java/doc/usersguide/md/CodeDuplicationMetricsRef.md b/OpenStaticAnalyzer/java/doc/usersguide/md/CodeDuplicationMetricsRef.md
@@ -36,11 +36,14 @@ The following table summarizes the code duplication metrics, their abbreviations
   Clone Classes                     CCL                                     X        X       X           X      X         X       
   Clone Complexity                  CCO      X                X             X        X       X           X      X         X       
   Clone Coverage                    CC                                      X        X       X           X      X         X       
+  Clone Elimination Effort          CEE                       X                                                           X       
+  Clone Elimination Gain            CEG                       X                                                           X       
   Clone Embeddedness                CE       X                X                                                                   
   Clone Instances                   CI                        X             X        X       X           X      X         X       
   Clone Line Coverage               CLC                                     X        X       X           X      X         X       
   Clone Lines of Code               CLLOC    X                X                                                                   
   Clone Logical Line Coverage       CLLC                                    X        X       X           X      X         X       
+  Clone Risk                        CR                        X                                                           X       
   Clone Variability                 CV       X                X                                                                   
   Lines of Duplicated Code          LDC                                     X        X       X           X      X         X       
   Logical Lines of Duplicated Code  LLDC                                    X        X       X           X      X         X       
@@ -74,6 +77,18 @@ The following table summarizes the code duplication metrics, their abbreviations
 
 **Component:** ratio of code covered by code duplications in the component to the size of the component, expressed in terms of the number of syntactic entities (statements, expressions, etc.).
 
+#### Clone Elimination Effort (CEE) {#CEE}
+
+**Clone class:** index of the effort required to eliminate the clone class. It is computed as the product of CI, CE, and NCR.
+
+**Component:** index of the effort required to eliminate all clones from the component. It is computed as the sum of CEE of the clone classes in the component.
+
+#### Clone Elimination Gain (CEG) {#CEG}
+
+**Clone class:** index of the gain resulting from eliminating the clone class. It is computed as the ratio of CR to CEE.
+
+**Component:** index of the gain resulting from eliminating all clones from the component. It is computed as the logistic function of the ratio of CR to CEE.
+
 #### Clone Embeddedness (CE) {#CE}
 
 **Clone instance:** sum of incoming and outgoing references (function calls, variable references, type references; different references to the same entity are counted only once) in the code fragment corresponding to the clone instance, weighted with the number of directory changes between the referenced code fragments.
@@ -106,6 +121,12 @@ The following table summarizes the code duplication metrics, their abbreviations
 
 **Component:** ratio of code covered by code duplications in the component to the size of the component, expressed in terms of logical lines of code (non-empty, non-comment lines).
 
+#### Clone Risk (CR) {#CR}
+
+**Clone class:** risk index of the existence of the clone class. It is computed as the product of CLLOC, CI, CCO, NCR, and CV.
+
+**Component:** relative risk index of the existence of code duplications in the component. It is computed as the sum of CR of the clone classes in the component, divided by the total logical lines of code (non-empty, non-comment lines) of the component. It expresses the risk index projected to a non-empty, non-comment line of code in the component.
+
 #### Clone Variability (CV) {#CV}
 
 **Clone instance:** instability of the clone instance since it appeared. It is computed as the ratio of the number of previously analyzed revisions when the instance had been changed to its age (CA).