diff --git a/.travis.yml b/.travis.yml index aa1eb942..713662e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,17 @@ sudo: required language: scala jdk: - openjdk8 -scala: - - 2.11.2 install: - wget http://download.redis.io/releases/redis-5.0.1.tar.gz - tar -xzvf redis-5.0.1.tar.gz - make -C redis-5.0.1 -j4 - export PATH=$PWD/redis-5.0.1/src:$PATH -script: make test +script: + - make test # test with scala 2.11 + - sleep 5s # let redis exit gracefully (we use kill, not kill -9 in makefile) + - ps aux | grep redis + - ./dev/change-scala-version.sh 2.12 # switch to scala 2.12 + - make test # test with scala 2.12 cache: directories: - $HOME/.m2 diff --git a/dev/change-scala-version.sh b/dev/change-scala-version.sh new file mode 100755 index 00000000..e155a5c8 --- /dev/null +++ b/dev/change-scala-version.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -e + +VALID_VERSIONS=( 2.11 2.12 ) + +SCALA_211_MINOR_VERSION="12" +SCALA_212_MINOR_VERSION="9" + +usage() { + echo "Usage: $(basename $0) [-h|--help] +where : + -h| --help Display this help text + valid version values : ${VALID_VERSIONS[*]} +" 1>&2 + exit 1 +} + +if [[ ($# -ne 1) || ( $1 == "--help") || $1 == "-h" ]]; then + usage +fi + +TO_MAJOR_VERSION=$1 + +check_scala_version() { + for i in ${VALID_VERSIONS[*]}; do [ $i = "$1" ] && return 0; done + echo "Invalid Scala version: $1. Valid versions: ${VALID_VERSIONS[*]}" 1>&2 + exit 1 +} + +check_scala_version "$TO_MAJOR_VERSION" + +if [ $TO_MAJOR_VERSION = "2.12" ]; then + FROM_MAJOR_VERSION="2.11" + FROM_MINOR_VERSION=$SCALA_211_MINOR_VERSION + TO_MINOR_VERSION=$SCALA_212_MINOR_VERSION +else + FROM_MAJOR_VERSION="2.12" + FROM_MINOR_VERSION=$SCALA_212_MINOR_VERSION + TO_MINOR_VERSION=$SCALA_211_MINOR_VERSION +fi + +sed_i() { + sed -e "$1" "$2" > "$2.tmp" && mv "$2.tmp" "$2" +} + +export -f sed_i + +# change +BASEDIR=$(dirname $0)/.. +find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \ + -exec bash -c "sed_i 's/\(artifactId.*\)_'$FROM_MAJOR_VERSION'/\1_'$TO_MAJOR_VERSION'/g' {}" \; + +# change +find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \ + -exec bash -c "sed_i 's/\(\)'$FROM_MAJOR_VERSION'/\1'$TO_MAJOR_VERSION'/g' {}" \; + +# change +find "$BASEDIR" -name 'pom.xml' -not -path '*target*' -print \ + -exec bash -c "sed_i 's/\(.*\.\)'$FROM_MINOR_VERSION'/\1'$TO_MINOR_VERSION'/g' {}" \; + diff --git a/doc/dev.md b/doc/dev.md index 7c645758..26a1f2f9 100644 --- a/doc/dev.md +++ b/doc/dev.md @@ -25,3 +25,12 @@ To build Spark-Redis skipping tests, run: ``` mvn clean package -DskipTests ``` + +To change scala version use `./dev/change-scala-version.sh` script. It will change scala version in `pom.xml`. For example: +``` +./dev/change-scala-version.sh 2.12 +``` + +``` +./dev/change-scala-version.sh 2.11 +``` \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5971ac76..ff5b5556 100644 --- a/pom.xml +++ b/pom.xml @@ -2,8 +2,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 com.redislabs - spark-redis - 2.4.1-SNAPSHOT + spark-redis_2.11 + 2.4.2-SNAPSHOT Spark-Redis A Spark library for Redis http://github.com/RedisLabs/spark-redis @@ -300,7 +300,7 @@ org.scalatest scalatest_${scala.major.version} - 2.2.1 + 3.0.8 test diff --git a/src/main/scala/org/apache/spark/sql/redis/RedisSourceRelation.scala b/src/main/scala/org/apache/spark/sql/redis/RedisSourceRelation.scala index 5c7540e2..245e2967 100644 --- a/src/main/scala/org/apache/spark/sql/redis/RedisSourceRelation.scala +++ b/src/main/scala/org/apache/spark/sql/redis/RedisSourceRelation.scala @@ -132,7 +132,7 @@ class RedisSourceRelation(override val sqlContext: SQLContext, } // write data - data.foreachPartition { partition => + data.foreachPartition { partition: Iterator[Row] => // grouped iterator to only allocate memory for a portion of rows partition.grouped(iteratorGroupingSize).foreach { batch => // the following can be optimized to not create a map