Skip to content

Commit 28939cd

Browse files
louislepagelouislepage
authored
Merge branch 'apache:main' into shap-values
2 parents 71013cb + 9f82e9f commit 28939cd

File tree

343 files changed

+37007
-9284
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

343 files changed

+37007
-9284
lines changed

‎.github/workflows/documentation.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,13 +67,13 @@ jobs:
6767
uses: actions/checkout@v4
6868

6969
- name: Setup Python
70-
uses: actions/setup-python@v4
70+
uses: actions/setup-python@v5
7171
with:
7272
python-version: 3.7
7373
architecture: 'x64'
7474

7575
- name: Cache Pip Dependencies
76-
uses: actions/cache@v3
76+
uses: actions/cache@v4
7777
with:
7878
path: ~/.cache/pip
7979
key: ${{ runner.os }}-pip-docs-${{ hashFiles('src/main/python/docs/requires-docs.txt') }}

‎.github/workflows/javaTests.yml

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ on:
4444
jobs:
4545
java_tests:
4646
runs-on: ${{ matrix.os }}
47-
timeout-minutes: 90
47+
timeout-minutes: 30
4848
strategy:
4949
fail-fast: false
5050
matrix:
@@ -55,13 +55,19 @@ jobs:
5555
"**.component.c**.**",
5656
"**.component.e**.**,**.component.f**.**,**.component.m**.**",
5757
"**.component.p**.**,**.component.t**.**",
58-
"**.functions.a**.**,**.functions.binary.frame.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**",
58+
"**.functions.a**.**,**.functions.binary.matrix.**,**.functions.binary.scalar.**,**.functions.binary.tensor.**",
5959
"**.functions.blocks.**,**.functions.data.rand.**,",
60-
"**.functions.countDistinct.**,**.functions.countDistinctApprox.**,**.functions.data.misc.**,**.functions.lineage.**",
60+
"**.functions.countDistinct.**,**.functions.countDistinctApprox.**",
61+
"**.functions.data.misc.**,**.functions.lineage.**",
6162
"**.functions.compress.**,**.functions.data.tensor.**,**.functions.codegenalg.parttwo.**,**.functions.codegen.**,**.functions.caching.**",
6263
"**.functions.binary.matrix_full_cellwise.**,**.functions.binary.matrix_full_other.**",
6364
"**.functions.federated.algorithms.**,**.functions.federated.io.**,**.functions.federated.paramserv.**",
64-
"**.functions.federated.primitives.**,**.functions.federated.transform.**",
65+
"**.functions.federated.transform.**",
66+
"**.functions.federated.primitives.part1.** -Dtest-threadCount=1 -Dtest-forkCount=1",
67+
"**.functions.federated.primitives.part2.** -Dtest-threadCount=1 -Dtest-forkCount=1",
68+
"**.functions.federated.primitives.part3.** -Dtest-threadCount=1 -Dtest-forkCount=1",
69+
"**.functions.federated.primitives.part4.** -Dtest-threadCount=1 -Dtest-forkCount=1",
70+
"**.functions.federated.primitives.part5.** -Dtest-threadCount=1 -Dtest-forkCount=1",
6571
"**.functions.federated.monitoring.**,**.functions.federated.multitenant.**",
6672
"**.functions.federated.codegen.**,**.functions.federated.FederatedTestObjectConstructor",
6773
"**.functions.codegenalg.partone.**",
@@ -79,12 +85,12 @@ jobs:
7985
"**.functions.reorg.**,**.functions.rewrite.**,**.functions.ternary.**,**.functions.transform.**",
8086
"**.functions.unary.matrix.**,**.functions.linearization.**,**.functions.jmlc.**"
8187
]
88+
java: [11]
8289
name: ${{ matrix.tests }}
8390
steps:
8491
- name: Checkout Repository
8592
uses: actions/checkout@v4
8693

87-
8894
- name: ${{ matrix.tests }}
8995
uses: ./.github/action/
9096
id: test
@@ -96,13 +102,12 @@ jobs:
96102
ARTIFACT_NAME="transient_jacoco"
97103
ARTIFACT_NAME+="-${{ matrix.os }}"
98104
ARTIFACT_NAME+="-java-${{ matrix.java }}"
99-
ARTIFACT_NAME+="-${{ matrix.javadist }}"
100105
ARTIFACT_NAME+="-${{ matrix.tests }}"
101106
ARTIFACT_NAME=${ARTIFACT_NAME//\*/x} # replace * with x
102107
echo "ARTIFACT_NAME=$ARTIFACT_NAME" >> $GITHUB_ENV
103108
104109
- name: Save Java Test Coverage as Artifact
105-
uses: actions/upload-artifact@v3
110+
uses: actions/upload-artifact@v4
106111
with:
107112
name: ${{ env.ARTIFACT_NAME }}
108113
path: target/jacoco.exec
@@ -119,15 +124,15 @@ jobs:
119124
uses: actions/checkout@v4
120125

121126
- name: Cache Maven Dependencies
122-
uses: actions/cache@v3
127+
uses: actions/cache@v4
123128
with:
124129
path: ~/.m2/repository
125130
key: ${{ runner.os }}-maven-test-${{ hashFiles('**/pom.xml') }}
126131
restore-keys: |
127132
${{ runner.os }}-maven-test-
128133
129134
- name: Download all Jacoco Artifacts
130-
uses: actions/download-artifact@v3
135+
uses: actions/download-artifact@v4
131136
with:
132137
path: target
133138

@@ -142,23 +147,23 @@ jobs:
142147

143148
- name: Upload Jacoco Report Artifact PR
144149
if: (github.repository_owner == 'apache') && (github.ref_name != 'main')
145-
uses: actions/upload-artifact@v3
150+
uses: actions/upload-artifact@v4
146151
with:
147152
name: Java Code Coverage (Jacoco)
148153
path: target/site/jacoco
149154
retention-days: 7
150155

151156
- name: Upload Jacoco Report Artifact Main
152157
if: (github.repository_owner == 'apache') && (github.ref_name == 'main')
153-
uses: actions/upload-artifact@v3
158+
uses: actions/upload-artifact@v4
154159
with:
155160
name: Java Code Coverage (Jacoco)
156161
path: target/site/jacoco
157162
retention-days: 30
158163

159164
- name: Upload Jacoco Report Artifact Fork
160165
if: (github.repository_owner != 'apache')
161-
uses: actions/upload-artifact@v3
166+
uses: actions/upload-artifact@v4
162167
with:
163168
name: Java Code Coverage (Jacoco)
164169
path: target/site/jacoco

‎.github/workflows/python.yml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,21 +66,21 @@ jobs:
6666
cache: 'maven'
6767

6868
- name: Cache Pip Dependencies
69-
uses: actions/cache@v3
69+
uses: actions/cache@v4
7070
with:
7171
path: ~/.cache/pip
7272
key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('src/main/python/setup.py') }}
7373

7474
- name: Cache Datasets
75-
uses: actions/cache@v3
75+
uses: actions/cache@v4
7676
with:
7777
path: |
7878
src/main/python/systemds/examples/tutorials/mnist
7979
src/main/python/systemds/examples/tutorials/adult/data.zip
8080
key: ${{ runner.os }}-mnist-${{ hashFiles('src/main/python/systemds/examples/tutorials/mnist.py') }}-${{ hashFiles('src/main/python/systemds/examples/tutorials/adult.py') }}
8181

8282
- name: Cache Deb Dependencies
83-
uses: actions/cache@v3
83+
uses: actions/cache@v4
8484
with:
8585
path: /var/cache/apt/archives
8686
key: ${{ runner.os }}-${{ hashFiles('.github/workflows/python.yml') }}
@@ -89,7 +89,7 @@ jobs:
8989
run: mvn -ntp clean package -P distribution -B
9090

9191
- name: Setup Python ${{ matrix.python-version }}
92-
uses: actions/setup-python@v4
92+
uses: actions/setup-python@v5
9393
with:
9494
python-version: ${{ matrix.python-version }}
9595
architecture: 'x64'
@@ -112,13 +112,15 @@ jobs:
112112
export SYSTEMDS_ROOT=$(pwd)
113113
export PATH=$SYSTEMDS_ROOT/bin:$PATH
114114
export SYSDS_QUIET=1
115+
export LOG4JPROP=$SYSTEMDS_ROOT/src/test/resources/log4j.properties
115116
cd src/main/python
116117
unittest-parallel -t . -s tests
117118
# python -m unittest discover -s tests -p 'test_*.py'
118119
echo "Exit Status: " $?
119120
120121
- name: Run all python tests no environment
121122
run: |
123+
export LOG4JPROP=$(pwd)/src/test/resources/log4j.properties
122124
cd src/main/python
123125
unittest-parallel -t . -s tests
124126
# python -m unittest discover -s tests -p 'test_*.py'

‎.github/workflows/release-scripts.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
- run: printf "JAVA_HOME = $JAVA_HOME \n"
5555

5656
- name: Cache local Maven repository
57-
uses: actions/cache@v3
57+
uses: actions/cache@v4
5858
with:
5959
path: ~/.m2/repository
6060
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
.externalToolBuilders/
3333
maven-eclipse.xml
3434
/src/main/cpp/kernels/.cproject
35+
/src/main/cpp/log
3536

3637
# Intellij
3738
.idea/
@@ -77,8 +78,10 @@ docs/_site
7778
# Test Artifacts
7879
src/test/scripts/**/*.dmlt
7980
src/test/scripts/functions/mlcontextin/
81+
src/test/scripts/functions/frame/io/
8082
src/test/java/org/apache/sysds/test/component/compress/io/files
8183
src/test/java/org/apache/sysds/test/component/compress/io/filesIOSpark/*
84+
src/test/java/org/apache/sysds/test/component/compress/io/filesIOTest
8285
.factorypath
8386

8487
# Excluded sources

‎dev/release/src/test/java/org/apache/sysds/validation/ValidateLicAndNotice.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ public ValidateLicAndNotice(String strDistroDir) {
8888
}
8989

9090
public String getDistroDir() {
91-
return strDistroDir;
91+
return strDistroDir;
9292
}
9393

9494
public void setDistroDir(String strDistroDir) {
@@ -286,7 +286,7 @@ public static HashMap<String, Boolean> getPackagesFromZip (String zipFileName) t
286286
while(e.hasMoreElements()) {
287287
entry = (ZipEntry) e.nextElement();
288288
if(! entry.getName().startsWith(Constants.SYSTEMDS_PACKAGE) &&
289-
entry.getName().endsWith("." + Constants.CLASS)) {
289+
entry.getName().endsWith("." + Constants.CLASS)) {
290290
int iPos = entry.getName().lastIndexOf("/");
291291
if (iPos > 0) {
292292
String strPackageName = entry.getName().substring(0, iPos);
@@ -422,7 +422,7 @@ private List<String> getZipsInDistro(File directory) {
422422
*/
423423
private static String readFile(File file) throws java.io.IOException {
424424
StringBuffer fileData = new StringBuffer();
425-
BufferedReader reader = new BufferedReader(new FileReader(file));
425+
BufferedReader reader = new BufferedReader(new FileReader(file));
426426
char[] buf = new char[1024];
427427
int numRead = 0;
428428
while ((numRead = reader.read(buf)) != -1) {
@@ -544,7 +544,7 @@ public static boolean extractFileFromTGZ (String tgzFileName, String fileName, S
544544
bufOut = new BufferedOutputStream(fos, Constants.BUFFER);
545545
while ((count = bufIn.read(data, 0, Constants.BUFFER)) != -1) {
546546
bufOut.write(data, 0, count);
547-
}
547+
}
548548
bufOut.flush();
549549
bufOut.close();
550550
bufIn.close();
@@ -589,7 +589,7 @@ public static List<String> getFilesFromZip (String zipFileName, String fileExt)
589589
if(entry.getName().endsWith("." + fileExt)) {
590590
int iPos = entry.getName().lastIndexOf("/");
591591
if (iPos == 0)
592-
--iPos;
592+
--iPos;
593593
String strFileName = entry.getName().substring(iPos+1);
594594
files.add(strFileName);
595595
}

‎docker/entrypoint.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ cd /github/workspace/src/main/cpp
2626
./build.sh
2727
cd /github/workspace
2828

29-
export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=128m"
29+
export MAVEN_OPTS="-Xmx512m"
3030

3131
log="/tmp/sysdstest.log"
3232
mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building SystemDS"

‎docker/testsysds.Dockerfile

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,7 @@ RUN apt-get install -y --no-install-recommends \
7373
libssl-dev \
7474
r-base \
7575
r-base-dev \
76-
r-base-core\
77-
76+
r-base-core \
7877
&& Rscript installDependencies.R \
7978
&& rm -rf installDependencies.R \
8079
&& rm -rf /var/lib/apt/lists/*

‎pom.xml

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,11 +39,11 @@
3939
</licenses>
4040

4141
<properties>
42-
<hadoop.version>3.3.4</hadoop.version>
42+
<hadoop.version>3.3.6</hadoop.version>
4343
<antlr.version>4.8</antlr.version>
44-
<protobuf.version>3.20.3</protobuf.version>
45-
<spark.version>3.3.1</spark.version>
46-
<scala.version>2.12.0</scala.version>
44+
<protobuf.version>3.23.4</protobuf.version>
45+
<spark.version>3.5.0</spark.version>
46+
<scala.version>2.12.18</scala.version>
4747
<scala.binary.version>2.12</scala.binary.version>
4848
<maven.build.timestamp.format>yyyy-MM-dd HH:mm:ss z</maven.build.timestamp.format>
4949
<project.build.outputTimestamp>1</project.build.outputTimestamp>
@@ -1369,7 +1369,7 @@
13691369
<dependency>
13701370
<groupId>com.fasterxml.jackson.core</groupId>
13711371
<artifactId>jackson-databind</artifactId>
1372-
<version>2.13.4.2</version>
1372+
<version>2.15.2</version>
13731373
</dependency>
13741374

13751375
<dependency>
@@ -1405,7 +1405,7 @@
14051405
<dependency>
14061406
<groupId>org.codehaus.janino</groupId>
14071407
<artifactId>janino</artifactId>
1408-
<version>3.0.16</version>
1408+
<version>3.1.9</version>
14091409
<scope>provided</scope>
14101410
</dependency>
14111411

@@ -1438,7 +1438,7 @@
14381438
<dependency>
14391439
<groupId>io.netty</groupId>
14401440
<artifactId>netty-all</artifactId>
1441-
<version>4.1.68.Final</version>
1441+
<version>4.1.96.Final</version>
14421442
<!-- <scope>provided</scope> -->
14431443
<exclusions>
14441444
<exclusion>
@@ -1477,17 +1477,17 @@
14771477
</dependency>
14781478
<dependency>
14791479
<groupId>org.slf4j</groupId>
1480-
<artifactId>jul-to-slf4j</artifactId>
1480+
<artifactId>slf4j-reload4j</artifactId>
14811481
<version>${slf4j.version}</version>
14821482
</dependency>
14831483
<dependency>
14841484
<groupId>org.slf4j</groupId>
1485-
<artifactId>jcl-over-slf4j</artifactId>
1485+
<artifactId>jul-to-slf4j</artifactId>
14861486
<version>${slf4j.version}</version>
14871487
</dependency>
14881488
<dependency>
14891489
<groupId>org.slf4j</groupId>
1490-
<artifactId>slf4j-reload4j</artifactId>
1490+
<artifactId>jcl-over-slf4j</artifactId>
14911491
<version>${slf4j.version}</version>
14921492
</dependency>
14931493

‎scripts/builtin/dist.dml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@
3232
# -----------------------------------------------------------------------------------------------
3333

3434
m_dist = function(Matrix[Double] X) return (Matrix[Double] Y) {
35-
G = X %*% t(X);
36-
Y = sqrt(-2 * G + outer(diag(G), t(diag(G)), "+"));
35+
n = nrow(X)
36+
s = rowSums(X^2)
37+
Y = sqrt(-2 * X %*% t(X) + s + t(s))
3738
Y = replace(target = Y, pattern=NaN, replacement = 0);
3839
}

0 commit comments

Comments
 (0)