Update catboost

fomichev3000 · Aug 1, 2017 · 648a46b · 648a46b
1 parent 6810256
commit 648a46b
Show file tree

Hide file tree

Showing 258 changed files with 6,631 additions and 3,935 deletions.
diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@
 [Documentation](https://tech.yandex.com/catboost/doc/dg/concepts/about-docpage/) |
 [Installation](https://tech.yandex.com/catboost/doc/dg/concepts/cli-installation-docpage/)
 
-[![Build Status](https://travis-ci.org/catboost/catboost.svg?branch=master)](https://travis-ci.org/catboost/catboost)
+[![Build Status](https://travis-ci.org/catboost/catboost.svg?branch=master)](https://travis-ci.org/catboost/catboost) 
 [![PyPI version](https://badge.fury.io/py/catboost.svg)](https://badge.fury.io/py/catboost)
 
 CatBoost is a machine learning method based on gradient boosting over decision trees.

diff --git a/build/mapping.conf.json b/build/mapping.conf.json
@@ -1,15 +1,17 @@
 {
     "tasks": {
+        "133697471": {
+            "win32": "312651669", 
+            "darwin": "312651618", 
+            "linux": "312651700"
+        }, 
         "74052635": {
             "win32": "173608136", 
             "linux-aarch64": "173608113", 
             "darwin": "173608117", 
             "cygwin": "173608101", 
             "linux": "173608153"
         }, 
-        "48027726": {
-            "linux": "103177511"
-        }, 
         "123001907": {
             "Darwin-14.5.0-x86_64-i386-64bit": "289123862", 
             "Linux-4.4.52-25-x86_64-with-Ubuntu-10.04-lucid": "289123926"
@@ -22,32 +24,30 @@
             "darwin": "303191680", 
             "linux": "303191750"
         }, 
+        "48027726": {
+            "linux": "103177511"
+        }, 
         "127828138": {
             "Darwin-14.5.0-x86_64-i386-64bit": "300008394"
         }, 
-        "130175340": {
-            "win32": "305233024", 
-            "darwin": "305233000", 
-            "linux": "305233045"
-        }, 
         "70007402": {}
     }, 
     "resources": {
         "289123862": "http://storage.mds.yandex.net/get-devtools-opensource/471749/289123862", 
-        "299229708": "http://storage.mds.yandex.net/get-devtools-opensource/250854/299229708", 
-        "289123926": "http://storage.mds.yandex.net/get-devtools-opensource/479623/289123926", 
-        "103177511": "http://storage.mds.yandex.net/get-devtools-opensource/479623/103177511", 
-        "305233024": "http://storage.mds.yandex.net/get-devtools-opensource/479623/305233024", 
-        "173608136": "http://storage.mds.yandex.net/get-devtools-opensource/233854/173608136", 
-        "305233000": "http://storage.mds.yandex.net/get-devtools-opensource/233854/305233000", 
+        "299229708": "http://storage.mds.yandex.net/get-devtools-opensource/471749/299229708", 
+        "289123926": "http://storage.mds.yandex.net/get-devtools-opensource/233854/289123926", 
+        "103177511": "http://storage.mds.yandex.net/get-devtools-opensource/471749/103177511", 
+        "312651618": "http://storage.mds.yandex.net/get-devtools-opensource/233854/312651618", 
+        "173608136": "http://storage.mds.yandex.net/get-devtools-opensource/471749/173608136", 
+        "312651669": "http://storage.mds.yandex.net/get-devtools-opensource/373962/312651669", 
         "303191750": "http://storage.mds.yandex.net/get-devtools-opensource/471749/303191750", 
+        "312651700": "http://storage.mds.yandex.net/get-devtools-opensource/471749/312651700", 
         "173608153": "http://storage.mds.yandex.net/get-devtools-opensource/233854/173608153", 
         "300008394": "http://storage.mds.yandex.net/get-devtools-opensource/250854/300008394", 
-        "173608101": "http://storage.mds.yandex.net/get-devtools-opensource/233854/173608101", 
-        "173608113": "http://storage.mds.yandex.net/get-devtools-opensource/471749/173608113", 
+        "173608101": "http://storage.mds.yandex.net/get-devtools-opensource/471749/173608101", 
+        "173608113": "http://storage.mds.yandex.net/get-devtools-opensource/479623/173608113", 
         "303191680": "http://storage.mds.yandex.net/get-devtools-opensource/233854/303191680", 
         "303191729": "http://storage.mds.yandex.net/get-devtools-opensource/250854/303191729", 
-        "173608117": "http://storage.mds.yandex.net/get-devtools-opensource/233854/173608117", 
-        "305233045": "http://storage.mds.yandex.net/get-devtools-opensource/233854/305233045"
+        "173608117": "http://storage.mds.yandex.net/get-devtools-opensource/479623/173608117"
     }
 }
diff --git a/build/plugins/ytest.py b/build/plugins/ytest.py
@@ -195,6 +195,7 @@ def onadd_ytest(unit, *args):
         'TEST-CWD': unit.get('TEST_CWD_VALUE') or '',
         'FUZZ-DICTS': serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE')),
         'FUZZ-OPTS': serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE')),
+        'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '',
     }
 
     # use all cores if fuzzing requested
@@ -273,7 +274,6 @@ def onadd_check(unit, *args):
             raise Exception('{} is not allowed in LINT(), use one of {}'.format(check_level, allowed_levels.keys()))
         flat_args[1] = allowed_levels[check_level]
         script_rel_path = "java.style"
-        unit.oninternal_recurse("devtools/jstyle-runner")
     else:
         script_rel_path = check_type
 
@@ -457,6 +457,7 @@ def onjava_test(unit, *args):
         'JVM_ARGS': serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE')),
         'SYSTEM_PROPERTIES': props,
         'TEST-CWD': test_cwd,
+        'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '',
     }
 
     data = dump_test(test_record)
@@ -555,6 +556,7 @@ def _dump_test(
             'OLD_PYTEST': 'yes' if old_pytest else 'no',
             'PYTHON-PATHS': serialize_list(python_paths),
             'TEST-CWD': test_cwd or '',
+            'SKIP_TEST': unit.get('SKIP_TEST_VALUE') or '',
         }
         if binary_path:
             test_record['BINARY-PATH'] = strip_roots(binary_path)

diff --git a/build/scripts/run_sonar.py b/build/scripts/run_sonar.py
@@ -64,7 +64,7 @@ def main(opts, props_args):
         opts.java_binary_path,
         '-classpath',
         opts.sonar_scanner_jar_path,
-    ] + base_props_args + props_args + [opts.sonar_scanner_main_class]
+    ] + base_props_args + props_args + [opts.sonar_scanner_main_class, '-X']
 
     p = sp.Popen(sonar_cmd, stdout=sp.PIPE, stderr=sp.STDOUT)
     out, _ = p.communicate()

diff --git a/build/ya.conf.json b/build/ya.conf.json
@@ -562,6 +562,44 @@
                 ]
             }
         },
+        "clang39aarch64": {
+            "name": "clang39",
+            "tools": {
+                "cc": { "bottle": "clang39", "executable": "cc" },
+                "c++": { "bottle": "clang39", "executable": "c++" },
+                "gcov": { "bottle": "clang39", "executable": "gcov" },
+                "c++filt": { "bottle": "clang39", "executable": "c++filt" },
+                "llvm-symbolizer": { "bottle": "clang39", "executable": "llvm-symbolizer" },
+                "llvm-cov": { "bottle": "clang39", "executable": "llvm-cov" },
+                "nm": { "bottle": "clang39", "executable": "nm" },
+                "strip": { "bottle": "clang39", "executable": "strip" }
+            },
+            "platforms": [
+                {"host": {"os": "LINUX"}, "target": {"os": "LINUX", "arch": "aarch64"}, "default": true}
+            ],
+            "params": {
+                "type": "clang",
+                "match_root": "CLANG",
+                "werror_mode": "all",
+                "c_compiler": "$(CLANG)/bin/clang",
+                "cxx_compiler": "$(CLANG)/bin/clang++",
+                "ar": "$(CLANG)/bin/llvm-ar",
+                "llvm-symbolizer": "$(CLANG)/bin/llvm-symbolizer",
+                "gcc_version": "3.9",
+                "target_opt": ["--target=aarch64-linux-gnu", "--sysroot=$(OS_SDK_ROOT)", "-B$(OS_SDK_ROOT)/usr/bin", "-march=armv8a"],
+                "platform": {
+                    "LINUX": [
+                        "contrib/libs/platform/linux"
+                    ]
+                }
+            },
+            "env": {
+                "LD_LIBRARY_PATH": [
+                    "$(CLANG)/lib",
+                    "$(OS_SDK_ROOT)/usr/lib/x86_64-linux-gnu"
+                ]
+            }
+        },
         "clang40": {
             "tools": {
                 "cc": { "bottle": "clang40", "executable": "cc" },
@@ -1673,6 +1711,17 @@
                   {"host": {"os": "DARWIN"}, "default": true}
             ]
         },
+        "jstyle_runner": {
+            "tools": {
+                "jstyle_runner": { "bottle": "jstyle_runner" }
+            },
+            "platforms": [
+                  {"host": {"os": "LINUX"}, "default": true},
+                  {"host": {"os": "FREEBSD"}, "default": true},
+                  {"host": {"os": "WIN"}, "default": true},
+                  {"host": {"os": "DARWIN"}, "default": true}
+            ]
+        },
         "kiwi_protos": {
             "tools": {
                 "kiwi_protos": { "bottle": "kiwi_protos" }
@@ -1792,7 +1841,7 @@
         },
         "test_tool": {
             "formula": {
-                "sandbox_id": 130175340,
+                "sandbox_id": 133697471,
                 "match": "test_tool"
             },
             "executable": {
@@ -1828,7 +1877,7 @@
         },
         "bloat": {
             "formula": {
-                "sandbox_id": [49648326],
+                "sandbox_id": [132153728],
                 "match": "Bloat"
             },
             "executable": {
@@ -1980,6 +2029,12 @@
                 "match": "ERRORPRONE"
             }
         },
+        "jstyle_runner": {
+            "formula": {
+                "sandbox_id": [131011094],
+                "match": "JSTYLERUNNER"
+            }
+        },
         "wine": {
             "formula": {
                 "sandbox_id": 42480153,
@@ -1991,7 +2046,7 @@
         },
         "deploy": {
             "formula": {
-                "sandbox_id": 117076601,
+                "sandbox_id": 132426558,
                 "match": "DEPLOY"
             },
             "executable": {
@@ -2018,7 +2073,7 @@
         },
         "ytyndexer": {
             "formula": {
-                "sandbox_id": 129716924,
+                "sandbox_id": 131259617,
                 "match": "YTYNDEXER"
             },
             "executable": {

diff --git a/build/ymake.core.conf b/build/ymake.core.conf
@@ -1,4 +1,4 @@
-FAKEID=r2983760
+FAKEID=r3006696
 
 CURDIR=.
 BINDIR=bin:/
@@ -8,6 +8,10 @@ ECHO=echo
 # Only to fix warning from library/build_info
 SANDBOX_TASK_ID=0
 
+when ($ARCH_AARCH64) {
+    OS_SDK=ubuntu-16
+}
+
 when (!$OS_SDK) {
     OS_SDK=system
 }
@@ -503,6 +507,20 @@ module PROGRAM: LINK_UNIT {
     }
 }
 
+macro YCR_GENERATE_CONFIGS_INTL(Package, App, Configs...) {
+    MACRO_PROP(CMD ${tool:"maps/tools/yacare/scripts"} $Package $App ${cwd:BINDIR} ${output;hide;pre=debian/$Package/:Configs})
+}
+
+macro YCR_GENERATE_CONFIGS(Package, App) {
+    MACRO_PROP(CMD $YCR_GENERATE_CONFIGS_INTL($Package $App ${pre=etc/logrotate.d/:App} ${pre=etc/monrun/conf.d/;suf=-alive.conf:App} ${pre=etc/monrun/conf.d/;suf=-load.conf:App} ${pre=etc/nginx/sites-available/:App.conf} ${pre=etc/syslog-ng/conf-available/:App.conf} ${pre=usr/lib/yandex/maps/yacare/checks/:App} ${pre=usr/lib/yandex/maps/yacare/stat/:App}))
+}
+
+YCR_LINK_EXE=$LINK_EXE && $YCR_GENERATE_CONFIGS($REALPRJNAME $REALPRJNAME)
+
+module YCR_PROGRAM: PROGRAM {
+    MODULE_PROP(CMD YCR_LINK_EXE)
+}
+
 macro EXPORTS_SCRIPT(Arg) {
     SET(EXPORTS_FILE $Arg)
 }
@@ -586,6 +604,11 @@ macro PYTHON_PATH(Path) {
     SET(TEST_PYTHON_PATH_VALUE $Path)
 }
 
+SKIP_TEST_VALUE=
+macro SKIP_TEST(Reason) {
+    SET(SKIP_TEST_VALUE $Reason)
+}
+
 LINT_LEVEL_VALUE=base
 macro NO_LINT() {
     SET(LINT_LEVEL_VALUE none)
@@ -985,8 +1008,8 @@ module _JAVA_LIBRARY: BASE_UNIT {
 }
 
 JAVA_PROTOBUF=contrib/java/com/google/protobuf/protobuf-java/3.2.0
-JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.4.0
-JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.4.0
+JAVA_GRPC_STUB=contrib/java/io/grpc/grpc-stub/1.5.0
+JAVA_GRPC_PROTOBUF=contrib/java/io/grpc/grpc-protobuf/1.5.0
 
 module JAVA_PROTO_LIBRARY: _JAVA_LIBRARY {
     MODULE_PROP(EXTS .jsrc)

diff --git a/catboost/R-package/DESCRIPTION b/catboost/R-package/DESCRIPTION
@@ -7,3 +7,4 @@ Imports: jsonlite
 License: Apache License, Version 2.0
 Description: This is CatBoost R package. An open-source gradient boosting on decision trees library with categorical features support out of the box. 
 Author: CatBoost Developers
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -7,3 +7,4 @@ Imports: jsonlite
		License: Apache License, Version 2.0
		Description: This is CatBoost R package. An open-source gradient boosting on decision trees library with categorical features support out of the box.
		Author: CatBoost Developers