From e0f0eb4f4cbe920a840581c93f8c4a1aa5b2755f Mon Sep 17 00:00:00 2001 From: "M.Schmidt" Date: Tue, 21 Jan 2025 11:22:39 +0100 Subject: [PATCH 1/4] refactor `AnalysisInputLocation`s `Collection getClassSources(...)` to `Stream getClassSources(...) -> passing the Autoclosable object to the callers to avoid collecting streams and re-stream() them in the meantime --- .../frontend/ApkAnalysisInputLocation.java | 10 +- .../core/frontend/AbstractClassSource.java | 12 +- .../sootup/core/frontend/SootClassSource.java | 2 +- .../inputlocation/AnalysisInputLocation.java | 4 +- .../inputlocation/EagerInputLocation.java | 5 +- .../ScopedAnalysisInputLocation.java | 11 +- .../ArchiveBasedAnalysisInputLocation.java | 5 +- .../ClassFileBasedAnalysisInputLocation.java | 91 +++++ .../DefaultRuntimeAnalysisInputLocation.java | 4 +- .../DirectoryBasedAnalysisInputLocation.java | 48 +++ .../JavaClassPathAnalysisInputLocation.java | 15 +- .../JavaModulePathAnalysisInputLocation.java | 13 +- .../JrtFileSystemAnalysisInputLocation.java | 10 +- ...eMultiReleaseJarAnalysisInputLocation.java | 4 +- .../MultiReleaseJarAnalysisInputLocation.java | 8 +- .../OTFCompileAnalysisInputLocation.java | 3 +- .../PathBasedAnalysisInputLocation.java | 349 +----------------- .../WarArchiveAnalysisInputLocation.java | 235 ++++++++++++ .../bytecode/frontend/InfiniteLoopsTest.java | 4 +- .../java/bytecode/frontend/Soot1577Test.java | 4 +- .../TryWithResourcesFinallyTests.java | 5 +- .../AnalysisInputLocationTest.java | 5 +- ...vaModulePathAnalysisInputLocationTest.java | 3 +- ...rtFileSystemAnalysisInputLocationTest.java | 8 +- .../PathBasedAnalysisInputLocationTest.java | 5 +- .../frontend/interceptors/AggregatorTest.java | 6 +- .../interceptors/CopyPropagatorTest.java | 4 +- .../DeadAssignmentEliminatorTest.java | 6 +- .../core/ModuleInfoAnalysisInputLocation.java | 4 +- .../java/core/views/JavaModuleView.java | 16 +- .../java/sootup/java/core/views/JavaView.java | 11 +- .../JavaSourcePathAnalysisInputLocation.java | 15 +- .../JavaSourcePathNamespaceTest.java | 11 +- .../MinimalSourceTestSuiteBase.java | 2 +- .../frontend/JimpleAnalysisInputLocation.java | 16 +- .../JimpleStringAnalysisInputLocation.java | 8 +- .../sootup/jimple/frontend/JimpleView.java | 2 +- 37 files changed, 492 insertions(+), 472 deletions(-) create mode 100644 sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java create mode 100644 sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java create mode 100644 sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java diff --git a/sootup.apk.frontend/src/main/java/sootup/apk/frontend/ApkAnalysisInputLocation.java b/sootup.apk.frontend/src/main/java/sootup/apk/frontend/ApkAnalysisInputLocation.java index 9870dcc79fe..5eba6ea4a5e 100644 --- a/sootup.apk.frontend/src/main/java/sootup/apk/frontend/ApkAnalysisInputLocation.java +++ b/sootup.apk.frontend/src/main/java/sootup/apk/frontend/ApkAnalysisInputLocation.java @@ -25,9 +25,8 @@ import java.io.IOException; import java.nio.file.Path; import java.util.*; -import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import org.jf.dexlib2.iface.DexFile; import sootup.apk.frontend.Util.*; import sootup.apk.frontend.dexpler.DexClassProvider; @@ -104,17 +103,16 @@ private Optional getClassSourceInternal( @Nonnull @Override - public Collection getClassSources(@Nonnull View view) { + public Stream getClassSources(@Nonnull View view) { return classNamesList.entrySet().stream() .flatMap( className -> StreamUtils.optionalToStream( getClassSource( - view.getIdentifierFactory().getClassType(className.getKey()), view))) - .collect(Collectors.toList()); + view.getIdentifierFactory().getClassType(className.getKey()), view))); } - @Nullable + @Nonnull @Override public SourceType getSourceType() { return SourceType.Application; diff --git a/sootup.core/src/main/java/sootup/core/frontend/AbstractClassSource.java b/sootup.core/src/main/java/sootup/core/frontend/AbstractClassSource.java index 3205dfd0f83..ec6a9d2d5f4 100644 --- a/sootup.core/src/main/java/sootup/core/frontend/AbstractClassSource.java +++ b/sootup.core/src/main/java/sootup/core/frontend/AbstractClassSource.java @@ -39,17 +39,17 @@ public abstract class AbstractClassSource { // holds information about the class - protected final AnalysisInputLocation classSource; + protected final AnalysisInputLocation analysisInputLocation; // holds information about the specific data unit where the information about a class is stored protected final Path sourcePath; // the classType that identifies the containing class information protected ClassType classSignature; public AbstractClassSource( - @Nonnull AnalysisInputLocation classSource, + @Nonnull AnalysisInputLocation analysisInputLocation, @Nonnull ClassType classSignature, @Nonnull Path sourcePath) { - this.classSource = classSource; + this.analysisInputLocation = analysisInputLocation; this.classSignature = classSignature; this.sourcePath = sourcePath; } @@ -65,7 +65,7 @@ public ClassType getClassType() { } public AnalysisInputLocation getAnalysisInputLocation() { - return classSource; + return analysisInputLocation; } public Path getSourcePath() { @@ -88,12 +88,12 @@ public boolean equals(@Nullable Object o) { return false; } AbstractClassSource that = (AbstractClassSource) o; - return Objects.equal(classSource, that.classSource) + return Objects.equal(analysisInputLocation, that.analysisInputLocation) && Objects.equal(sourcePath, that.sourcePath); } @Override public int hashCode() { - return Objects.hashCode(classSource, sourcePath); + return Objects.hashCode(analysisInputLocation, sourcePath); } } diff --git a/sootup.core/src/main/java/sootup/core/frontend/SootClassSource.java b/sootup.core/src/main/java/sootup/core/frontend/SootClassSource.java index 41865ac6d9f..59a085a3e64 100644 --- a/sootup.core/src/main/java/sootup/core/frontend/SootClassSource.java +++ b/sootup.core/src/main/java/sootup/core/frontend/SootClassSource.java @@ -64,7 +64,7 @@ public SootClassSource( } protected SootClassSource(SootClassSource delegate) { - super(delegate.classSource, delegate.getClassType(), delegate.getSourcePath()); + super(delegate.analysisInputLocation, delegate.getClassType(), delegate.getSourcePath()); } /** Reads from the source to retrieve its methods. This may be an expensive operation. */ diff --git a/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java b/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java index a8cc3e7190a..aecc5648ae1 100644 --- a/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java +++ b/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java @@ -21,9 +21,9 @@ * #L% */ -import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.frontend.SootClassSource; import sootup.core.model.SootClass; @@ -61,7 +61,7 @@ public interface AnalysisInputLocation { * @return The source entries. */ @Nonnull - Collection getClassSources(@Nonnull View view); + Stream getClassSources(@Nonnull View view); /** * If the AnalysisInputLocation is initialized with the SourceType then this method should return diff --git a/sootup.core/src/main/java/sootup/core/inputlocation/EagerInputLocation.java b/sootup.core/src/main/java/sootup/core/inputlocation/EagerInputLocation.java index 366afe42026..26b0ce27622 100644 --- a/sootup.core/src/main/java/sootup/core/inputlocation/EagerInputLocation.java +++ b/sootup.core/src/main/java/sootup/core/inputlocation/EagerInputLocation.java @@ -22,6 +22,7 @@ */ import com.google.common.collect.ImmutableMap; import java.util.*; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import sootup.core.frontend.SootClassSource; @@ -69,9 +70,9 @@ public EagerInputLocation( @Nonnull @Override - public Collection getClassSources(@Nullable View view) { + public Stream getClassSources(@Nullable View view) { // FIXME: add classloadingoptions - return map.values(); + return map.values().stream(); } @Nonnull diff --git a/sootup.core/src/main/java/sootup/core/inputlocation/ScopedAnalysisInputLocation.java b/sootup.core/src/main/java/sootup/core/inputlocation/ScopedAnalysisInputLocation.java index ddf7cf13799..15b15ea79c4 100644 --- a/sootup.core/src/main/java/sootup/core/inputlocation/ScopedAnalysisInputLocation.java +++ b/sootup.core/src/main/java/sootup/core/inputlocation/ScopedAnalysisInputLocation.java @@ -22,10 +22,9 @@ * #L% */ -import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.frontend.SootClassSource; import sootup.core.model.SourceType; @@ -60,12 +59,8 @@ public Optional getClassSource( @Nonnull @Override - public Collection getClassSources(@Nonnull View view) { - // possibility to streamify this method to apply the filter at earlier stage i.e. before - // creating the ClassSources would be a faster approach.. - return inputLocation.getClassSources(view).stream() - .filter(type -> filter(type.getClassType())) - .collect(Collectors.toList()); + public Stream getClassSources(@Nonnull View view) { + return inputLocation.getClassSources(view).filter(type -> filter(type.getClassType())); } @Nonnull diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java index d092ed27309..b698c5bb7c8 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java @@ -32,6 +32,7 @@ import java.nio.file.Path; import java.util.*; import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.model.SourceType; import sootup.core.transform.BodyInterceptor; @@ -99,6 +100,7 @@ public Optional getClassSource(@Nonnull ClassType type, @No try { FileSystem fs = fileSystemCache.get(path); final Path archiveRoot = fs.getPath("/"); + // TODO: dont create a new AsmJavaClassProvider all the time! return getClassSourceInternal( (JavaClassType) type, archiveRoot, new AsmJavaClassProvider(view)); } catch (ExecutionException e) { @@ -108,10 +110,11 @@ public Optional getClassSource(@Nonnull ClassType type, @No @Override @Nonnull - public Collection getClassSources(@Nonnull View view) { + public Stream getClassSources(@Nonnull View view) { try { FileSystem fs = fileSystemCache.get(path); final Path archiveRoot = fs.getPath("/"); + // TODO: dont create a new AsmJavaClassProvider all the time! return walkDirectory( archiveRoot, view.getIdentifierFactory(), new AsmJavaClassProvider(view)); } catch (ExecutionException e) { diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java new file mode 100644 index 00000000000..3e516f6def5 --- /dev/null +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java @@ -0,0 +1,91 @@ +package sootup.java.bytecode.frontend.inputlocation; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import org.apache.commons.io.FilenameUtils; +import sootup.core.IdentifierFactory; +import sootup.core.model.SourceType; +import sootup.core.transform.BodyInterceptor; +import sootup.core.types.ClassType; +import sootup.core.views.View; +import sootup.interceptors.BytecodeBodyInterceptors; +import sootup.java.bytecode.frontend.conversion.AsmJavaClassProvider; +import sootup.java.core.JavaSootClassSource; +import sootup.java.core.types.JavaClassType; + +public class ClassFileBasedAnalysisInputLocation extends PathBasedAnalysisInputLocation { + + @Nonnull private final String omittedPackageName; + + public ClassFileBasedAnalysisInputLocation( + @Nonnull Path classFilePath, + @Nonnull String omittedPackageName, + @Nonnull SourceType srcType) { + this( + classFilePath, + omittedPackageName, + srcType, + BytecodeBodyInterceptors.Default.getBodyInterceptors()); + } + + public ClassFileBasedAnalysisInputLocation( + @Nonnull Path classFilePath, + @Nonnull String omittedPackageName, + @Nonnull SourceType srcType, + @Nonnull List bodyInterceptors) { + super(classFilePath, srcType, bodyInterceptors); + this.omittedPackageName = omittedPackageName; + + if (!Files.isRegularFile(classFilePath)) { + throw new IllegalArgumentException("Needs to point to a regular file!"); + } + + if (Files.isDirectory(classFilePath)) { + throw new IllegalArgumentException("Needs to point to a regular file - not to a directory."); + } + } + + @Override + @Nonnull + public Optional getClassSource(@Nonnull ClassType type, @Nonnull View view) { + + if (!type.getPackageName().getName().startsWith(omittedPackageName)) { + return Optional.empty(); + } + + return getSingleClass((JavaClassType) type, path, new AsmJavaClassProvider(view)); + } + + @Nonnull + @Override + public Stream getClassSources(@Nonnull View view) { + AsmJavaClassProvider classProvider = new AsmJavaClassProvider(view); + IdentifierFactory factory = view.getIdentifierFactory(); + Path dirPath = this.path.getParent(); + + final String fullyQualifiedName = fromPath(dirPath, path); + + Optional javaSootClassSource = + classProvider + .createClassSource(this, path, factory.getClassType(fullyQualifiedName)) + .map(src -> (JavaSootClassSource) src); + + return Stream.of(javaSootClassSource.get()); + } + + @Nonnull + protected String fromPath(@Nonnull Path baseDirPath, Path packageNamePathAndClass) { + String str = + FilenameUtils.removeExtension( + packageNamePathAndClass + .subpath(baseDirPath.getNameCount(), packageNamePathAndClass.getNameCount()) + .toString() + .replace(packageNamePathAndClass.getFileSystem().getSeparator(), ".")); + + return omittedPackageName.isEmpty() ? str : omittedPackageName + "." + str; + } +} diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DefaultRuntimeAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DefaultRuntimeAnalysisInputLocation.java index 76c2d741270..175b2bad27d 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DefaultRuntimeAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DefaultRuntimeAnalysisInputLocation.java @@ -22,9 +22,9 @@ * #L% */ -import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.frontend.SootClassSource; import sootup.core.inputlocation.AnalysisInputLocation; @@ -68,7 +68,7 @@ public Optional getClassSource( @Nonnull @Override - public Collection getClassSources(@Nonnull View view) { + public Stream getClassSources(@Nonnull View view) { return backingInputLocation.getClassSources(view); } diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java new file mode 100644 index 00000000000..64fdf9f283c --- /dev/null +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java @@ -0,0 +1,48 @@ +package sootup.java.bytecode.frontend.inputlocation; + +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import javax.annotation.Nonnull; +import sootup.core.model.SourceType; +import sootup.core.transform.BodyInterceptor; +import sootup.core.types.ClassType; +import sootup.core.views.View; +import sootup.java.bytecode.frontend.conversion.AsmJavaClassProvider; +import sootup.java.core.JavaSootClassSource; +import sootup.java.core.types.JavaClassType; + +class DirectoryBasedAnalysisInputLocation extends PathBasedAnalysisInputLocation { + + protected DirectoryBasedAnalysisInputLocation( + @Nonnull Path path, + @Nonnull SourceType srcType, + @Nonnull List bodyInterceptors) { + this(path, srcType, bodyInterceptors, Collections.emptyList()); + } + + protected DirectoryBasedAnalysisInputLocation( + @Nonnull Path path, + @Nonnull SourceType srcType, + @Nonnull List bodyInterceptors, + @Nonnull Collection ignoredPaths) { + super(path, srcType, bodyInterceptors, ignoredPaths); + } + + @Override + @Nonnull + public Stream getClassSources(@Nonnull View view) { + // FIXME: 1) store the classprovider reference as a field; 2) and above too; and 3) move view + // which is only used in SootNode to be just there? + return walkDirectory(path, view.getIdentifierFactory(), new AsmJavaClassProvider(view)); + } + + @Override + @Nonnull + public Optional getClassSource(@Nonnull ClassType type, @Nonnull View view) { + return getClassSourceInternal((JavaClassType) type, path, new AsmJavaClassProvider(view)); + } +} diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaClassPathAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaClassPathAnalysisInputLocation.java index e67b2ea92f4..f0583baf800 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaClassPathAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaClassPathAnalysisInputLocation.java @@ -168,20 +168,17 @@ public List getBodyInterceptors() { * @param entry A class path entry * @return A stream of class path entries with wildcards exploded */ - private static @Nonnull Stream handleWildCards(@Nonnull String entry) { + @Nonnull + private static Stream handleWildCards(@Nonnull String entry) { return handleWildCards(entry, FileSystems.getDefault()); } @Override @Nonnull - public Collection getClassSources(@Nonnull View view) { - // By using a set here, already added classes won't be overwritten and the class which is found - // first will be kept - Set found = new HashSet<>(); - for (AnalysisInputLocation inputLocation : cpEntries) { - found.addAll(inputLocation.getClassSources(view)); - } - return found.stream().map(src -> (JavaSootClassSource) src).collect(Collectors.toList()); + public Stream getClassSources(@Nonnull View view) { + return cpEntries.stream() + .flatMap(inputLocation -> inputLocation.getClassSources(view)) + .map(src -> (JavaSootClassSource) src); } @Override diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocation.java index 39adfa49eb3..e5e3016bc9b 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocation.java @@ -25,7 +25,6 @@ import java.nio.file.FileSystems; import java.nio.file.Path; import java.util.*; -import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.IdentifierFactory; @@ -107,16 +106,14 @@ public Set getModules(View view) { @Override @Nonnull - public Collection getClassSources(@Nonnull View view) { + public Stream getClassSources(@Nonnull View view) { IdentifierFactory identifierFactory = view.getIdentifierFactory(); Preconditions.checkArgument( identifierFactory instanceof JavaModuleIdentifierFactory, "Factory must be a JavaModuleSignatureFactory"); Collection allModules = moduleFinder.getAllModules(); - return allModules.stream() - .flatMap(sig -> getClassSourcesInternal(sig, view)) - .collect(Collectors.toList()); + return allModules.stream().flatMap(sig -> getClassSourcesInternal(sig, view)); } @Nonnull @@ -133,13 +130,13 @@ public List getBodyInterceptors() { @Override @Nonnull - public Collection getModulesClassSources( + public Stream getModulesClassSources( @Nonnull ModuleSignature moduleSignature, @Nonnull View view) { IdentifierFactory identifierFactory = view.getIdentifierFactory(); Preconditions.checkArgument( identifierFactory instanceof JavaModuleIdentifierFactory, "Factory must be a JavaModuleSignatureFactory"); - return getClassSourcesInternal(moduleSignature, view).collect(Collectors.toList()); + return getClassSourcesInternal(moduleSignature, view); } protected Stream getClassSourcesInternal( @@ -150,7 +147,7 @@ protected Stream getClassSourcesInternal( return Stream.empty(); } - return inputLocation.getClassSources(view).stream().map(src -> (JavaSootClassSource) src); + return inputLocation.getClassSources(view).map(src -> (JavaSootClassSource) src); } @Override diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocation.java index 25a0d46686c..57c7090f61f 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocation.java @@ -131,10 +131,9 @@ public Optional getClassSource( /** Retreive CLassSources of a module specified by methodSignature */ @Override @Nonnull - public Collection getModulesClassSources( + public Stream getModulesClassSources( @Nonnull ModuleSignature moduleSignature, @Nonnull View view) { - return getClassSourcesInternal(moduleSignature, view.getIdentifierFactory(), view) - .collect(Collectors.toList()); + return getClassSourcesInternal(moduleSignature, view.getIdentifierFactory(), view); } @Nonnull @@ -182,12 +181,11 @@ protected ClassProvider getClassProvider(@Nonnull View view) { } @Override - public @Nonnull Collection getClassSources(@Nonnull View view) { + public @Nonnull Stream getClassSources(@Nonnull View view) { Collection moduleSignatures = discoverModules(); return moduleSignatures.stream() - .flatMap(sig -> getClassSourcesInternal(sig, view.getIdentifierFactory(), view)) - .collect(Collectors.toList()); + .flatMap(sig -> getClassSourcesInternal(sig, view.getIdentifierFactory(), view)); } /** diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ModuleMultiReleaseJarAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ModuleMultiReleaseJarAnalysisInputLocation.java index 07cb41fca49..41d98b5b242 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ModuleMultiReleaseJarAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ModuleMultiReleaseJarAnalysisInputLocation.java @@ -23,11 +23,11 @@ */ import java.nio.file.Path; -import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.frontend.SootClassSource; import sootup.core.model.SourceType; @@ -109,7 +109,7 @@ protected ModuleInfoAnalysisInputLocation createAnalysisInputLocation( } @Override - public Collection getModulesClassSources( + public Stream getModulesClassSources( @Nonnull ModuleSignature moduleSignature, @Nonnull View view) { // TODO: check if we need to combine modules as well or if only versioned .class files are return ((JavaModulePathAnalysisInputLocation) inputLocations.get(DEFAULT_VERSION)) diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/MultiReleaseJarAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/MultiReleaseJarAnalysisInputLocation.java index 8260556dc25..1bfd7267646 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/MultiReleaseJarAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/MultiReleaseJarAnalysisInputLocation.java @@ -173,11 +173,10 @@ public Optional getClassSource(@Nonnull ClassType type, @No @Override @Nonnull - public Collection getClassSources(@Nonnull View view) { - + public Stream getClassSources(@Nonnull View view) { Collection classSources = new ArrayList<>(); inputLocations.values().stream() - .flatMap(location -> location.getClassSources(view).stream()) + .flatMap(location -> location.getClassSources(view)) .map(src -> (JavaSootClassSource) src) .forEach( cs -> { @@ -193,7 +192,8 @@ public Collection getClassSources(@Nonnull View view) { } }); - return classSources; + // TODO: return stream directly + return classSources.stream(); } @Nonnull diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/OTFCompileAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/OTFCompileAnalysisInputLocation.java index be1b4626368..5f59eb69879 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/OTFCompileAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/OTFCompileAnalysisInputLocation.java @@ -33,6 +33,7 @@ import java.nio.file.attribute.FileTime; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.tools.*; import sootup.core.frontend.SootClassSource; @@ -95,7 +96,7 @@ public Optional getClassSource( @Nonnull @Override - public Collection getClassSources(@Nonnull View view) { + public Stream getClassSources(@Nonnull View view) { return inputLocation.getClassSources(view); } diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java index 07e63e0812b..dbd61a9a065 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java @@ -5,18 +5,8 @@ import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import javax.annotation.Nonnull; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.io.FilenameUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; import sootup.core.IdentifierFactory; import sootup.core.frontend.ClassProvider; import sootup.core.frontend.SootClassSource; @@ -24,12 +14,8 @@ import sootup.core.inputlocation.FileType; import sootup.core.model.SourceType; import sootup.core.transform.BodyInterceptor; -import sootup.core.types.ClassType; import sootup.core.util.PathUtils; import sootup.core.util.StreamUtils; -import sootup.core.views.View; -import sootup.interceptors.BytecodeBodyInterceptors; -import sootup.java.bytecode.frontend.conversion.AsmJavaClassProvider; import sootup.java.core.*; import sootup.java.core.types.JavaClassType; @@ -155,16 +141,18 @@ public static PathBasedAnalysisInputLocation create( + "' has to be pointing to the root of a class container, e.g. directory, jar, zip, apk, war etc."); } + /** returns a resource that must be closed! */ @Nonnull - Collection walkDirectory( + protected Stream walkDirectory( @Nonnull Path dirPath, @Nonnull IdentifierFactory factory, @Nonnull ClassProvider classProvider) { final FileType handledFileType = classProvider.getHandledFileType(); final String moduleInfoFilename = JavaModuleIdentifierFactory.MODULE_INFO_FILE + ".class"; - try (final Stream walk = Files.walk(dirPath)) { - return walk.filter( + try { + return Files.walk(dirPath) + .filter( filePath -> PathUtils.hasExtension(filePath, handledFileType) && !filePath.toString().endsWith(moduleInfoFilename) @@ -178,9 +166,7 @@ Collection walkDirectory( classProvider.createClassSource( this, p, factory.getClassType(fullyQualifiedName))); }) - .map(src -> (JavaSootClassSource) src) - .collect(Collectors.toList()); - + .map(src -> (JavaSootClassSource) src); } catch (IOException e) { throw new IllegalArgumentException(e); } @@ -226,327 +212,4 @@ protected Optional getSingleClass( return classSource.map(src -> (JavaSootClassSource) src); } - - public static class ClassFileBasedAnalysisInputLocation extends PathBasedAnalysisInputLocation { - - @Nonnull private final String omittedPackageName; - - public ClassFileBasedAnalysisInputLocation( - @Nonnull Path classFilePath, - @Nonnull String omittedPackageName, - @Nonnull SourceType srcType) { - this( - classFilePath, - omittedPackageName, - srcType, - BytecodeBodyInterceptors.Default.getBodyInterceptors()); - } - - public ClassFileBasedAnalysisInputLocation( - @Nonnull Path classFilePath, - @Nonnull String omittedPackageName, - @Nonnull SourceType srcType, - @Nonnull List bodyInterceptors) { - super(classFilePath, srcType, bodyInterceptors); - this.omittedPackageName = omittedPackageName; - - if (!Files.isRegularFile(classFilePath)) { - throw new IllegalArgumentException("Needs to point to a regular file!"); - } - - if (Files.isDirectory(classFilePath)) { - throw new IllegalArgumentException( - "Needs to point to a regular file - not to a directory."); - } - } - - @Override - @Nonnull - public Optional getClassSource( - @Nonnull ClassType type, @Nonnull View view) { - - if (!type.getPackageName().getName().startsWith(omittedPackageName)) { - return Optional.empty(); - } - - return getSingleClass((JavaClassType) type, path, new AsmJavaClassProvider(view)); - } - - @Nonnull - @Override - public Collection getClassSources(@Nonnull View view) { - AsmJavaClassProvider classProvider = new AsmJavaClassProvider(view); - IdentifierFactory factory = view.getIdentifierFactory(); - Path dirPath = this.path.getParent(); - - final String fullyQualifiedName = fromPath(dirPath, path); - - Optional classSource = - classProvider - .createClassSource(this, path, factory.getClassType(fullyQualifiedName)) - .map(src -> (JavaSootClassSource) src); - return Collections.singletonList(classSource.get()); - } - - @Nonnull - protected String fromPath(@Nonnull Path baseDirPath, Path packageNamePathAndClass) { - String str = - FilenameUtils.removeExtension( - packageNamePathAndClass - .subpath(baseDirPath.getNameCount(), packageNamePathAndClass.getNameCount()) - .toString() - .replace(packageNamePathAndClass.getFileSystem().getSeparator(), ".")); - - return omittedPackageName.isEmpty() ? str : omittedPackageName + "." + str; - } - } - - private static class DirectoryBasedAnalysisInputLocation extends PathBasedAnalysisInputLocation { - - protected DirectoryBasedAnalysisInputLocation( - @Nonnull Path path, - @Nonnull SourceType srcType, - @Nonnull List bodyInterceptors) { - this(path, srcType, bodyInterceptors, Collections.emptyList()); - } - - protected DirectoryBasedAnalysisInputLocation( - @Nonnull Path path, - @Nonnull SourceType srcType, - @Nonnull List bodyInterceptors, - @Nonnull Collection ignoredPaths) { - super(path, srcType, bodyInterceptors, ignoredPaths); - } - - @Override - @Nonnull - public Collection getClassSources(@Nonnull View view) { - // FIXME: 1) store the classprovider reference as a field; 2) and above too; and 3) move view - // which is only used in SootNode to be just there? - return walkDirectory(path, view.getIdentifierFactory(), new AsmJavaClassProvider(view)); - } - - @Override - @Nonnull - public Optional getClassSource( - @Nonnull ClassType type, @Nonnull View view) { - return getClassSourceInternal((JavaClassType) type, path, new AsmJavaClassProvider(view)); - } - } - - private static final class WarArchiveAnalysisInputLocation - extends DirectoryBasedAnalysisInputLocation { - public List containedInputLocations = new ArrayList<>(); - public static int maxAllowedBytesToExtract = - 1024 * 1024 * 500; // limit of extracted file size to protect against archive bombs - - private WarArchiveAnalysisInputLocation(@Nonnull Path warPath, @Nonnull SourceType srcType) - throws IOException { - this( - warPath, - srcType, - BytecodeBodyInterceptors.Default.getBodyInterceptors(), - Collections.emptyList()); - } - - private WarArchiveAnalysisInputLocation( - @Nonnull Path warPath, - @Nonnull SourceType srcType, - @Nonnull List bodyInterceptors, - @Nonnull Collection ignoredPaths) - throws IOException { - super( - Files.createTempDirectory("sootUp-war-" + warPath.hashCode()).toAbsolutePath(), - srcType, - bodyInterceptors, - ignoredPaths); - - extractWarFile(warPath, path); - - Path webInfPath = path.resolve("WEB-INF"); - // directorystructre as specified in SRV.9.5 of - // https://download.oracle.com/otn-pub/jcp/servlet-2.4-fr-spec-oth-JSpec/servlet-2_4-fr-spec.pdf?AuthParam=1625059899_16c705c72f7db7f85a8a7926558701fe - Path classDir = webInfPath.resolve("classes"); - if (Files.exists(classDir)) { - containedInputLocations.add( - new DirectoryBasedAnalysisInputLocation(classDir, srcType, bodyInterceptors)); - } - - Path libDir = webInfPath.resolve("lib"); - if (Files.exists(libDir)) { - try (Stream paths = Files.walk(libDir)) { - paths - .filter(f -> PathUtils.hasExtension(f, FileType.JAR)) - .forEach( - f -> - containedInputLocations.add( - new ArchiveBasedAnalysisInputLocation(f, srcType, bodyInterceptors))); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - - public WarArchiveAnalysisInputLocation( - Path path, SourceType srcType, List bodyInterceptors) throws IOException { - this(path, srcType, bodyInterceptors, Collections.emptyList()); - } - - @Override - @Nonnull - public Collection getClassSources(@Nonnull View view) { - - Set foundClasses = new HashSet<>(); - - for (AnalysisInputLocation inputLoc : containedInputLocations) { - foundClasses.addAll(inputLoc.getClassSources(view)); - } - return foundClasses.stream() - .map(src -> (JavaSootClassSource) src) - .collect(Collectors.toList()); - } - - @Override - @Nonnull - public Optional getClassSource( - @Nonnull ClassType type, @Nonnull View view) { - - for (AnalysisInputLocation inputLocation : containedInputLocations) { - final Optional classSource = - inputLocation.getClassSource(type, view); - if (classSource.isPresent()) { - return classSource.map(src -> (JavaSootClassSource) src); - } - } - - return Optional.empty(); - } - - /** - * Extracts the war file at the temporary location to analyze underlying class and jar files - * - *

[ms] hint: extracting is necessary to access nested (zip)filesystems with java8/java9 - * runtime - nested (zip)filesystems would work with java11 runtime (maybe java10) - * - * @param warFilePath The path to war file to be extracted - */ - void extractWarFile(Path warFilePath, final Path destDirectory) { - int extractedSize = 0; - try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(warFilePath))) { - File dest = destDirectory.toFile(); - if (!dest.exists()) { - if (!dest.mkdir()) { - throw new RuntimeException( - "Could not create the directory to extract Warfile: " + destDirectory); - } - dest.deleteOnExit(); - } - - ZipEntry zipEntry; - while ((zipEntry = zis.getNextEntry()) != null) { - Path filepath = destDirectory.resolve(zipEntry.getName()); - final File file = filepath.toFile(); - - String canonicalPathStr = file.getCanonicalPath(); - if (!canonicalPathStr.startsWith(destDirectory + File.separator)) { - throw new IllegalArgumentException( - "ZipSlip Attack Mitigated: ZipEntry points outside of the target dir: " - + file.getName()); - } - - file.deleteOnExit(); - if (zipEntry.isDirectory()) { - boolean mkdir = file.mkdir(); - if (!mkdir) { - throw new IllegalStateException( - "Could not create Directory: " + file.getAbsolutePath()); - } - } else { - byte[] incomingValues = new byte[4096]; - int readBytesZip; - if (file.exists()) { - // compare contents -> does it contain the extracted war already? - int readBytesExistingFile; - try (InputStream fis = Files.newInputStream(file.toPath()); - final BufferedInputStream bis = new BufferedInputStream(fis)) { - byte[] bisBuf = new byte[4096]; - while ((readBytesZip = zis.read(incomingValues)) != -1) { - if (extractedSize > maxAllowedBytesToExtract) { - throw new RuntimeException( - "The extracted warfile exceeds the size of " - + maxAllowedBytesToExtract - + " byte. Either the file is a big archive (-> increase PathBasedAnalysisInputLocation.WarArchiveInputLocation.maxAllowedBytesToExtract) or maybe it contains an archive bomb."); - } - readBytesExistingFile = bis.read(bisBuf, 0, readBytesZip); - if (readBytesExistingFile != readBytesZip) { - throw new RuntimeException( - "Can't extract File \"" - + file - + "\" as it already exists and has a different size."); - } else if (!Arrays.equals(bisBuf, incomingValues)) { - throw new RuntimeException( - "Can't extract File \"" - + file - + "\" as it already exists and has a different content which we can't override."); - } - extractedSize += readBytesZip; - } - } - - } else { - try (OutputStream fos = Files.newOutputStream(file.toPath()); - BufferedOutputStream bos = new BufferedOutputStream(fos); ) { - while ((readBytesZip = zis.read(incomingValues)) != -1) { - if (extractedSize > maxAllowedBytesToExtract) { - throw new RuntimeException( - "The extracted warfile exceeds the size of " - + maxAllowedBytesToExtract - + " byte. Either the file is a big archive or maybe it contains an archive bomb."); - } - bos.write(incomingValues, 0, readBytesZip); - extractedSize += readBytesZip; - } - } - } - } - zis.closeEntry(); - } - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Parses the web.xml file to search for the servlet-class classes in the extracted directory - * after the war file is extracted - * - *

[ms] helps to set entrypoints for analyses automatically (later) - * - * @param extractedWARPath The path where the war file is extracted Adds the classes associated - * to servlet-class in a {@link ArrayList} of {@link String} - */ - @Nonnull - public List retrieveServletClasses(String extractedWARPath) { - List classesInXML = new ArrayList<>(); - try { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = factory.newDocumentBuilder(); - Document document = builder.parse(new File(extractedWARPath + "/WEB-INF/web.xml")); - document.getDocumentElement().normalize(); - NodeList nList = document.getElementsByTagName("servlet"); - for (int temp = 0; temp < nList.getLength(); temp++) { - Node node = nList.item(temp); - if (node.getNodeType() == Node.ELEMENT_NODE) { - Element eElement = (Element) node; - classesInXML.add( - eElement.getElementsByTagName("servlet-class").item(0).getTextContent()); - } - } - } catch (ParserConfigurationException | SAXException | IOException e) { - throw new RuntimeException(e); - } - return classesInXML; - } - } } diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java new file mode 100644 index 00000000000..df05e9c46f3 --- /dev/null +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java @@ -0,0 +1,235 @@ +package sootup.java.bytecode.frontend.inputlocation; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.*; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; +import javax.annotation.Nonnull; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; +import sootup.core.frontend.SootClassSource; +import sootup.core.inputlocation.AnalysisInputLocation; +import sootup.core.inputlocation.FileType; +import sootup.core.model.SourceType; +import sootup.core.transform.BodyInterceptor; +import sootup.core.types.ClassType; +import sootup.core.util.PathUtils; +import sootup.core.views.View; +import sootup.interceptors.BytecodeBodyInterceptors; +import sootup.java.core.JavaSootClassSource; + +final class WarArchiveAnalysisInputLocation extends DirectoryBasedAnalysisInputLocation { + public List containedInputLocations = new ArrayList<>(); + public static int maxAllowedBytesToExtract = + 1024 * 1024 * 500; // limit of extracted file size to protect against archive bombs + + private WarArchiveAnalysisInputLocation(@Nonnull Path warPath, @Nonnull SourceType srcType) + throws IOException { + this( + warPath, + srcType, + BytecodeBodyInterceptors.Default.getBodyInterceptors(), + Collections.emptyList()); + } + + WarArchiveAnalysisInputLocation( + @Nonnull Path warPath, + @Nonnull SourceType srcType, + @Nonnull List bodyInterceptors, + @Nonnull Collection ignoredPaths) + throws IOException { + super( + Files.createTempDirectory("sootUp-war-" + warPath.hashCode()).toAbsolutePath(), + srcType, + bodyInterceptors, + ignoredPaths); + + extractWarFile(warPath, path); + + Path webInfPath = path.resolve("WEB-INF"); + // directorystructre as specified in SRV.9.5 of + // https://download.oracle.com/otn-pub/jcp/servlet-2.4-fr-spec-oth-JSpec/servlet-2_4-fr-spec.pdf?AuthParam=1625059899_16c705c72f7db7f85a8a7926558701fe + Path classDir = webInfPath.resolve("classes"); + if (Files.exists(classDir)) { + containedInputLocations.add( + new DirectoryBasedAnalysisInputLocation(classDir, srcType, bodyInterceptors)); + } + + Path libDir = webInfPath.resolve("lib"); + if (Files.exists(libDir)) { + try (Stream paths = Files.walk(libDir)) { + paths + .filter(f -> PathUtils.hasExtension(f, FileType.JAR)) + .forEach( + f -> + containedInputLocations.add( + new ArchiveBasedAnalysisInputLocation(f, srcType, bodyInterceptors))); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public WarArchiveAnalysisInputLocation( + Path path, SourceType srcType, List bodyInterceptors) throws IOException { + this(path, srcType, bodyInterceptors, Collections.emptyList()); + } + + @Override + @Nonnull + public Stream getClassSources(@Nonnull View view) { + return containedInputLocations.stream() + .flatMap(location -> location.getClassSources(view)) + .map(src -> (JavaSootClassSource) src); + } + + @Override + @Nonnull + public Optional getClassSource(@Nonnull ClassType type, @Nonnull View view) { + + for (AnalysisInputLocation inputLocation : containedInputLocations) { + final Optional classSource = + inputLocation.getClassSource(type, view); + if (classSource.isPresent()) { + return classSource.map(src -> (JavaSootClassSource) src); + } + } + + return Optional.empty(); + } + + /** + * Extracts the war file at the temporary location to analyze underlying class and jar files + * + *

[ms] hint: extracting is necessary to access nested (zip)filesystems with java8/java9 + * runtime - nested (zip)filesystems would work with java11 runtime (maybe java10) + * + * @param warFilePath The path to war file to be extracted + */ + void extractWarFile(Path warFilePath, final Path destDirectory) { + int extractedSize = 0; + try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(warFilePath))) { + File dest = destDirectory.toFile(); + if (!dest.exists()) { + if (!dest.mkdir()) { + throw new RuntimeException( + "Could not create the directory to extract Warfile: " + destDirectory); + } + dest.deleteOnExit(); + } + + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + Path filepath = destDirectory.resolve(zipEntry.getName()); + final File file = filepath.toFile(); + + String canonicalPathStr = file.getCanonicalPath(); + if (!canonicalPathStr.startsWith(destDirectory + File.separator)) { + throw new IllegalArgumentException( + "ZipSlip Attack Mitigated: ZipEntry points outside of the target dir: " + + file.getName()); + } + + file.deleteOnExit(); + if (zipEntry.isDirectory()) { + boolean mkdir = file.mkdir(); + if (!mkdir) { + throw new IllegalStateException( + "Could not create Directory: " + file.getAbsolutePath()); + } + } else { + byte[] incomingValues = new byte[4096]; + int readBytesZip; + if (file.exists()) { + // compare contents -> does it contain the extracted war already? + int readBytesExistingFile; + try (InputStream fis = Files.newInputStream(file.toPath()); + final BufferedInputStream bis = new BufferedInputStream(fis)) { + byte[] bisBuf = new byte[4096]; + while ((readBytesZip = zis.read(incomingValues)) != -1) { + if (extractedSize > maxAllowedBytesToExtract) { + throw new RuntimeException( + "The extracted warfile exceeds the size of " + + maxAllowedBytesToExtract + + " byte. Either the file is a big archive (-> increase PathBasedAnalysisInputLocation.WarArchiveInputLocation.maxAllowedBytesToExtract) or maybe it contains an archive bomb."); + } + readBytesExistingFile = bis.read(bisBuf, 0, readBytesZip); + if (readBytesExistingFile != readBytesZip) { + throw new RuntimeException( + "Can't extract File \"" + + file + + "\" as it already exists and has a different size."); + } else if (!Arrays.equals(bisBuf, incomingValues)) { + throw new RuntimeException( + "Can't extract File \"" + + file + + "\" as it already exists and has a different content which we can't override."); + } + extractedSize += readBytesZip; + } + } + + } else { + try (OutputStream fos = Files.newOutputStream(file.toPath()); + BufferedOutputStream bos = new BufferedOutputStream(fos); ) { + while ((readBytesZip = zis.read(incomingValues)) != -1) { + if (extractedSize > maxAllowedBytesToExtract) { + throw new RuntimeException( + "The extracted warfile exceeds the size of " + + maxAllowedBytesToExtract + + " byte. Either the file is a big archive or maybe it contains an archive bomb."); + } + bos.write(incomingValues, 0, readBytesZip); + extractedSize += readBytesZip; + } + } + } + } + zis.closeEntry(); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Parses the web.xml file to search for the servlet-class classes in the extracted directory + * after the war file is extracted + * + *

[ms] helps to set entrypoints for analyses automatically (later) + * + * @param extractedWARPath The path where the war file is extracted Adds the classes associated to + * servlet-class in a {@link ArrayList} of {@link String} + */ + @Nonnull + public List retrieveServletClasses(String extractedWARPath) { + List classesInXML = new ArrayList<>(); + try { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new File(extractedWARPath + "/WEB-INF/web.xml")); + document.getDocumentElement().normalize(); + NodeList nList = document.getElementsByTagName("servlet"); + for (int temp = 0; temp < nList.getLength(); temp++) { + Node node = nList.item(temp); + if (node.getNodeType() == Node.ELEMENT_NODE) { + Element eElement = (Element) node; + classesInXML.add(eElement.getElementsByTagName("servlet-class").item(0).getTextContent()); + } + } + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new RuntimeException(e); + } + return classesInXML; + } +} diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/InfiniteLoopsTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/InfiniteLoopsTest.java index 8ab669c7f53..1241d0d8189 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/InfiniteLoopsTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/InfiniteLoopsTest.java @@ -13,7 +13,7 @@ import sootup.core.model.Body; import sootup.core.model.SourceType; import sootup.core.signatures.MethodSignature; -import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.core.views.JavaView; @Tag(TestCategories.JAVA_8_CATEGORY) @@ -24,7 +24,7 @@ public class InfiniteLoopsTest { @Test public void test() { AnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( classFilePath, "", SourceType.Application, Collections.emptyList()); JavaView view = new JavaView(Collections.singletonList(inputLocation)); diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/Soot1577Test.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/Soot1577Test.java index 37c293f8913..43a39045a33 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/Soot1577Test.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/Soot1577Test.java @@ -9,7 +9,7 @@ import sootup.core.inputlocation.AnalysisInputLocation; import sootup.core.model.SootMethod; import sootup.core.model.SourceType; -import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.core.views.JavaView; @Tag(TestCategories.JAVA_8_CATEGORY) @@ -19,7 +19,7 @@ public class Soot1577Test { @Test public void test() { AnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( Paths.get("../shared-test-resources/soot-1577/g.class"), "cn.com.chinatelecom.account.api.c", SourceType.Application); diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java index 3153479079c..abb8788092c 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java @@ -9,8 +9,8 @@ import sootup.core.inputlocation.AnalysisInputLocation; import sootup.core.model.SourceType; import sootup.core.signatures.MethodSignature; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.core.util.printer.BriefStmtPrinter; -import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; import sootup.java.core.views.JavaView; @Tag(TestCategories.JAVA_8_CATEGORY) @@ -21,8 +21,7 @@ public class TryWithResourcesFinallyTests { @Test public void test() { AnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( - classFilePath, "", SourceType.Application); + new ClassFileBasedAnalysisInputLocation(classFilePath, "", SourceType.Application); JavaView view = new JavaView(Collections.singletonList(inputLocation)); MethodSignature methodSignature = diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/AnalysisInputLocationTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/AnalysisInputLocationTest.java index 93d0e9cfe00..04ec9b799ba 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/AnalysisInputLocationTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/AnalysisInputLocationTest.java @@ -5,7 +5,6 @@ import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collection; import java.util.List; import java.util.Optional; import sootup.core.IdentifierFactory; @@ -62,8 +61,6 @@ protected void testClassReceival( assertTrue(clazzOpt.isPresent()); assertEquals(classType, clazzOpt.get().getClassType()); } - final Collection classSources = ns.getClassSources(view); - - assertEquals(classSources.size(), classesFound); + assertEquals(ns.getClassSources(view).count(), classesFound); } } diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocationTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocationTest.java index 8f735c7e605..3a7f30085b7 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocationTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JavaModulePathAnalysisInputLocationTest.java @@ -125,8 +125,7 @@ public void testGetClassSources() { Collections.singletonList(inputLocation); JavaModuleView view = new JavaModuleView(inputLocations, moduleInfoAnalysisInputLocations); - final Collection classSources = inputLocation.getClassSources(view); - assertEquals(3, classSources.size()); + assertEquals(3, inputLocation.getClassSources(view).count()); } @Test diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java index 5c2a5804d4c..55dc8e30df7 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java @@ -7,6 +7,7 @@ import java.util.Collection; import java.util.Collections; import java.util.Optional; +import java.util.stream.Collectors; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import sootup.core.frontend.SootClassSource; @@ -46,12 +47,13 @@ public void getClassSources() { final ClassType sig2 = JavaModuleIdentifierFactory.getInstance().getClassType("System", "java.lang", "java.base"); - final Collection classSources = inputLocation.getClassSources(view); - assertTrue(classSources.size() > 26000); + final Collection classSources = + inputLocation.getClassSources(view).collect(Collectors.toList()); inputLocation.getClassSources(view); assertTrue( classSources.size() - > 20000); // not precise as this amount can differ depending on the included runtime + > 20000); // "a lot" not precise as this amount can differ depending on the included + // runtime // library assertTrue(classSources.stream().anyMatch(cs -> cs.getClassType().equals(sig1))); assertTrue(view.getClass(sig1).isPresent()); diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocationTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocationTest.java index 80819fdeb0e..53b153cbf2a 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocationTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocationTest.java @@ -58,8 +58,7 @@ public class PathBasedAnalysisInputLocationTest extends AnalysisInputLocationTes @Test public void testSingleClass() { PathBasedAnalysisInputLocation pathBasedNamespace = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( - cls, "", SourceType.Application); + new ClassFileBasedAnalysisInputLocation(cls, "", SourceType.Application); ArrayList sigs = new ArrayList<>(); sigs.add(getIdentifierFactory().getClassType("Employee")); testClassReceival(pathBasedNamespace, sigs, 1); @@ -77,7 +76,7 @@ public void testSingleClassDoesNotExist() { @Test public void testSingleClassWPackageName() { AnalysisInputLocation pathBasedNamespace = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( Paths.get("../shared-test-resources/ClassWithPackageName.class"), "ClassesPackageName", SourceType.Application); diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/AggregatorTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/AggregatorTest.java index dee5f95a731..a732e089391 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/AggregatorTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/AggregatorTest.java @@ -24,8 +24,8 @@ import sootup.core.types.ClassType; import sootup.core.types.PrimitiveType; import sootup.interceptors.Aggregator; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.bytecode.frontend.inputlocation.JavaClassPathAnalysisInputLocation; -import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; import sootup.java.core.JavaIdentifierFactory; import sootup.java.core.JavaSootMethod; import sootup.java.core.language.JavaJimple; @@ -183,7 +183,7 @@ public void testResource_Misuse() { public void testIssue739() { AnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( Paths.get("../shared-test-resources/bugfixes/Issue739_Aggregator.class"), "", SourceType.Application, @@ -203,7 +203,7 @@ public void testIssue739() { public void testIssue911() { AnalysisInputLocation inputLocationB = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( Paths.get("../shared-test-resources/bugfixes/Issue911_Aggregator.class"), "", SourceType.Application, diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/CopyPropagatorTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/CopyPropagatorTest.java index ab646bb8916..cd0cdb85b73 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/CopyPropagatorTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/CopyPropagatorTest.java @@ -33,8 +33,8 @@ import sootup.core.util.Utils; import sootup.core.views.View; import sootup.interceptors.CopyPropagator; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.bytecode.frontend.inputlocation.JavaClassPathAnalysisInputLocation; -import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; import sootup.java.core.JavaIdentifierFactory; import sootup.java.core.language.JavaJimple; import sootup.java.core.types.JavaClassType; @@ -405,7 +405,7 @@ private Body createExpectedCastExprBody() { @Test void testBigInput() { AnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( Paths.get("../shared-test-resources/bugfixes/SlowCopyPropagator.class"), "", SourceType.Application, diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/DeadAssignmentEliminatorTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/DeadAssignmentEliminatorTest.java index e7820f412a1..b39d2d68f57 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/DeadAssignmentEliminatorTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/DeadAssignmentEliminatorTest.java @@ -27,7 +27,7 @@ import sootup.core.util.Utils; import sootup.interceptors.DeadAssignmentEliminator; import sootup.interceptors.LocalPacker; -import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.core.JavaIdentifierFactory; import sootup.java.core.language.JavaJimple; import sootup.java.core.types.JavaClassType; @@ -173,7 +173,7 @@ private static Body.BodyBuilder createBody(boolean essentialOption) { @Test public void testDeadAssignmentEliminator() { AnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( classFilePath, "", SourceType.Application, @@ -233,7 +233,7 @@ public void testDeadAssignmentEliminator() { @Test public void testLocalCountAfterDAE() { AnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( classFilePath, "", SourceType.Application, diff --git a/sootup.java.core/src/main/java/sootup/java/core/ModuleInfoAnalysisInputLocation.java b/sootup.java.core/src/main/java/sootup/java/core/ModuleInfoAnalysisInputLocation.java index 5becf5e5217..061e642a600 100644 --- a/sootup.java.core/src/main/java/sootup/java/core/ModuleInfoAnalysisInputLocation.java +++ b/sootup.java.core/src/main/java/sootup/java/core/ModuleInfoAnalysisInputLocation.java @@ -22,9 +22,9 @@ * #L% */ -import java.util.Collection; import java.util.Optional; import java.util.Set; +import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.frontend.SootClassSource; import sootup.core.inputlocation.AnalysisInputLocation; @@ -38,7 +38,7 @@ */ public interface ModuleInfoAnalysisInputLocation extends AnalysisInputLocation { - Collection getModulesClassSources( + Stream getModulesClassSources( @Nonnull ModuleSignature moduleSignature, @Nonnull View view); @Nonnull diff --git a/sootup.java.core/src/main/java/sootup/java/core/views/JavaModuleView.java b/sootup.java.core/src/main/java/sootup/java/core/views/JavaModuleView.java index 9a057a9e237..3412bdded27 100644 --- a/sootup.java.core/src/main/java/sootup/java/core/views/JavaModuleView.java +++ b/sootup.java.core/src/main/java/sootup/java/core/views/JavaModuleView.java @@ -28,6 +28,7 @@ import javax.annotation.Nonnull; import sootup.core.cache.provider.ClassCacheProvider; import sootup.core.cache.provider.FullCacheProvider; +import sootup.core.frontend.SootClassSource; import sootup.core.inputlocation.AnalysisInputLocation; import sootup.core.signatures.PackageName; import sootup.core.types.ClassType; @@ -294,8 +295,9 @@ public synchronized Collection getModuleClasses( stream = inputLocations.stream() .flatMap( - input -> - input.getClassSources(this).stream().map(src -> (JavaSootClassSource) src)); + input -> { + return input.getClassSources(this).map(src -> (JavaSootClassSource) src); + }); } else { // named module @@ -307,7 +309,8 @@ public synchronized Collection getModuleClasses( .flatMap( input -> { // classpath - return input.getClassSources(this).stream() + return input + .getClassSources(this) .filter( cs -> moduleSignature.equals( @@ -319,7 +322,9 @@ public synchronized Collection getModuleClasses( .flatMap( input -> { // modulepath - return input.getModulesClassSources(moduleSignature, this).stream(); + Stream modulesClassSources = + input.getModulesClassSources(moduleSignature, this); + return modulesClassSources; })) .map(src -> (JavaSootClassSource) src); @@ -329,7 +334,8 @@ public synchronized Collection getModuleClasses( moduleInfoAnalysisInputLocations.stream() .flatMap( input -> - input.getModulesClassSources(moduleSignature, this).stream() + input + .getModulesClassSources(moduleSignature, this) .map(src -> (JavaSootClassSource) src)); } } diff --git a/sootup.java.core/src/main/java/sootup/java/core/views/JavaView.java b/sootup.java.core/src/main/java/sootup/java/core/views/JavaView.java index 54a0c032ca4..a861bf5e0ad 100644 --- a/sootup.java.core/src/main/java/sootup/java/core/views/JavaView.java +++ b/sootup.java.core/src/main/java/sootup/java/core/views/JavaView.java @@ -25,6 +25,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import sootup.core.cache.ClassCache; @@ -33,6 +34,7 @@ import sootup.core.cache.provider.FullCacheProvider; import sootup.core.frontend.AbstractClassSource; import sootup.core.inputlocation.AnalysisInputLocation; +import sootup.core.model.SootClass; import sootup.core.signatures.FieldSignature; import sootup.core.signatures.MethodSignature; import sootup.core.types.ClassType; @@ -87,11 +89,14 @@ public synchronized Stream getClasses() { Stream resolvedClasses = inputLocations.stream() - .flatMap(location -> location.getClassSources(this).stream()) + .flatMap( + location -> { + // TODO: [ms] find a way to not stream().collect().stream() + return location.getClassSources(this).collect(Collectors.toList()).stream(); + }) .map(this::buildClassFrom); isFullyResolved = true; - return resolvedClasses; } @@ -110,7 +115,7 @@ public synchronized Optional getClass(@Nonnull ClassType type) { @Nonnull public Optional getAnnotationClass(@Nonnull ClassType type) { - return getClass(type).filter(sc -> sc.isAnnotation()).map(sc -> (JavaAnnotationSootClass) sc); + return getClass(type).filter(SootClass::isAnnotation).map(sc -> (JavaAnnotationSootClass) sc); } @Override diff --git a/sootup.java.sourcecode.frontend/src/main/java/sootup/java/frontend/inputlocation/JavaSourcePathAnalysisInputLocation.java b/sootup.java.sourcecode.frontend/src/main/java/sootup/java/frontend/inputlocation/JavaSourcePathAnalysisInputLocation.java index 997613bfd7f..bf59138315b 100644 --- a/sootup.java.sourcecode.frontend/src/main/java/sootup/java/frontend/inputlocation/JavaSourcePathAnalysisInputLocation.java +++ b/sootup.java.sourcecode.frontend/src/main/java/sootup/java/frontend/inputlocation/JavaSourcePathAnalysisInputLocation.java @@ -24,6 +24,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.*; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.slf4j.Logger; @@ -131,15 +132,6 @@ public JavaSourcePathAnalysisInputLocation( this.exclusionFilePath = exclusionFilePath; this.classProvider = new WalaJavaClassProvider(sourcePaths, exclusionFilePath); this.bodyInterceptors = bodyInterceptors; - setSpecifiedAsBuiltInByUser(srcType); - } - - /** - * The method sets the value of the variable srcType. - * - * @param srcType the source type for the path can be Library, Application, Phantom. - */ - public void setSpecifiedAsBuiltInByUser(@Nullable SourceType srcType) { this.srcType = srcType; } @@ -157,9 +149,8 @@ public List getBodyInterceptors() { @Override @Nonnull - public Collection getClassSources(@Nonnull View view) { - - return classProvider.getClassSources(srcType); + public Stream getClassSources(@Nonnull View view) { + return classProvider.getClassSources(srcType).stream(); } @Override diff --git a/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/inputlocation/JavaSourcePathNamespaceTest.java b/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/inputlocation/JavaSourcePathNamespaceTest.java index 70088f5b33d..c41b1dd9c1d 100644 --- a/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/inputlocation/JavaSourcePathNamespaceTest.java +++ b/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/inputlocation/JavaSourcePathNamespaceTest.java @@ -3,16 +3,14 @@ import static org.junit.jupiter.api.Assertions.*; import java.util.Collection; -import java.util.HashSet; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import sootup.core.frontend.AbstractClassSource; import sootup.core.inputlocation.AnalysisInputLocation; -import sootup.core.model.SootClass; import sootup.core.model.SourceType; import sootup.core.signatures.PackageName; import sootup.core.types.ClassType; @@ -90,9 +88,8 @@ public void testInputSourcePathLibraryMode() { new JavaSourcePathAnalysisInputLocation(SourceType.Library, classPath); JavaView view = new JavaView(inputLocation); - Set classes = new HashSet<>(); // Set to track the classes to check - view.getClasses().filter(aClass -> !aClass.isLibraryClass()).forEach(classes::add); - - assertEquals(0, classes.size(), "User Defined class found, expected none"); + Stream javaSootClassStream = + view.getClasses().filter(aClass -> !aClass.isLibraryClass()); + assertEquals(0, javaSootClassStream.count(), "User Defined class found, expected none"); } } diff --git a/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/minimaltestsuite/MinimalSourceTestSuiteBase.java b/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/minimaltestsuite/MinimalSourceTestSuiteBase.java index 99b314dcedd..16f681242ba 100644 --- a/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/minimaltestsuite/MinimalSourceTestSuiteBase.java +++ b/sootup.java.sourcecode.frontend/src/test/java/sootup/java/frontend/minimaltestsuite/MinimalSourceTestSuiteBase.java @@ -83,7 +83,7 @@ public static String getTestDirectoryName(String classPath) { * respective name of the class */ public String getClassName(String classPath) { - System.out.println(classPath); + // System.out.println(classPath); String[] classPathArray = classPath.split("\\."); String className = classPathArray[classPathArray.length - 1].substring( diff --git a/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleAnalysisInputLocation.java b/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleAnalysisInputLocation.java index 6d558b20f6e..21394d89ab8 100644 --- a/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleAnalysisInputLocation.java +++ b/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleAnalysisInputLocation.java @@ -27,7 +27,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -89,14 +88,16 @@ public List getBodyInterceptors() { return bodyInterceptors; } + /** @return Autoclosable needs to be closed! */ @Nonnull - List walkDirectory( + Stream walkDirectory( @Nonnull Path dirPath, @Nonnull IdentifierFactory factory, @Nonnull ClassProvider classProvider) { - try (final Stream walk = Files.walk(path)) { - return walk.filter(filePath -> PathUtils.hasExtension(filePath, FileType.JIMPLE)) + try { + return Files.walk(path) + .filter(filePath -> PathUtils.hasExtension(filePath, FileType.JIMPLE)) .flatMap( p -> { String fullyQualifiedName = @@ -108,9 +109,7 @@ List walkDirectory( return StreamUtils.optionalToStream( classProvider.createClassSource( this, p, factory.getClassType(fullyQualifiedName))); - }) - .collect(Collectors.toList()); - + }); } catch (IOException e) { throw new IllegalArgumentException(e); } @@ -118,7 +117,8 @@ List walkDirectory( @Override @Nonnull - public Collection getClassSources(@Nonnull View view) { + public Stream getClassSources(@Nonnull View view) { + // TODO: dont create a new CLassProvider every time return walkDirectory( path, view.getIdentifierFactory(), new JimpleClassProvider(bodyInterceptors, view)); } diff --git a/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleStringAnalysisInputLocation.java b/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleStringAnalysisInputLocation.java index ff3fac31a46..8dc721d973a 100644 --- a/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleStringAnalysisInputLocation.java +++ b/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleStringAnalysisInputLocation.java @@ -24,10 +24,9 @@ import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import javax.annotation.Nonnull; import org.antlr.v4.runtime.CharStreams; import sootup.core.frontend.OverridingClassSource; @@ -91,9 +90,8 @@ public Optional getClassSource( @Nonnull @Override - public Collection getClassSources(@Nonnull View view) { - return Collections.singletonList( - getOverridingClassSource(jimpleFileContents, bodyInterceptors, view)); + public Stream getClassSources(@Nonnull View view) { + return Stream.of(getOverridingClassSource(jimpleFileContents, bodyInterceptors, view)); } @Nonnull diff --git a/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleView.java b/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleView.java index 966174d9d4a..4b9eaf8fe5e 100644 --- a/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleView.java +++ b/sootup.jimple.frontend/src/main/java/sootup/jimple/frontend/JimpleView.java @@ -153,7 +153,7 @@ private synchronized void resolveAll() { } inputLocations.stream() - .flatMap(location -> location.getClassSources(this).stream()) + .flatMap(location -> location.getClassSources(this)) .forEach(this::buildClassFrom); isFullyResolved = true; } From 1cc83f07ca07ae76047e4dfe83b424160d7f234d Mon Sep 17 00:00:00 2001 From: "M.Schmidt" Date: Tue, 21 Jan 2025 14:48:20 +0100 Subject: [PATCH 2/4] add doc --- .../java/sootup/core/inputlocation/AnalysisInputLocation.java | 2 +- .../inputlocation/ArchiveBasedAnalysisInputLocation.java | 1 + .../frontend/inputlocation/PathBasedAnalysisInputLocation.java | 2 +- .../java/bytecode/frontend/TryWithResourcesFinallyTests.java | 2 +- .../inputlocation/JrtFileSystemAnalysisInputLocationTest.java | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) diff --git a/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java b/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java index aecc5648ae1..4a4c5270c42 100644 --- a/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java +++ b/sootup.core/src/main/java/sootup/core/inputlocation/AnalysisInputLocation.java @@ -58,7 +58,7 @@ public interface AnalysisInputLocation { /** * Scan the input location and create ClassSources for every compilation / interpretation unit. * - * @return The source entries. + * @return an Autocloseable resource that must be closed! */ @Nonnull Stream getClassSources(@Nonnull View view); diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java index b698c5bb7c8..01f5978ae83 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ArchiveBasedAnalysisInputLocation.java @@ -108,6 +108,7 @@ public Optional getClassSource(@Nonnull ClassType type, @No } } + /** returns a Autocloseable resource that must be closed! */ @Override @Nonnull public Stream getClassSources(@Nonnull View view) { diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java index dbd61a9a065..6b26916616b 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/PathBasedAnalysisInputLocation.java @@ -141,7 +141,7 @@ public static PathBasedAnalysisInputLocation create( + "' has to be pointing to the root of a class container, e.g. directory, jar, zip, apk, war etc."); } - /** returns a resource that must be closed! */ + /** returns a Autocloseable resource that must be closed! */ @Nonnull protected Stream walkDirectory( @Nonnull Path dirPath, diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java index abb8788092c..e9d649e8368 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/TryWithResourcesFinallyTests.java @@ -9,8 +9,8 @@ import sootup.core.inputlocation.AnalysisInputLocation; import sootup.core.model.SourceType; import sootup.core.signatures.MethodSignature; -import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.core.util.printer.BriefStmtPrinter; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.core.views.JavaView; @Tag(TestCategories.JAVA_8_CATEGORY) diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java index 55dc8e30df7..0b02e58742f 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/inputlocation/JrtFileSystemAnalysisInputLocationTest.java @@ -53,7 +53,7 @@ public void getClassSources() { assertTrue( classSources.size() > 20000); // "a lot" not precise as this amount can differ depending on the included - // runtime + // runtime // library assertTrue(classSources.stream().anyMatch(cs -> cs.getClassType().equals(sig1))); assertTrue(view.getClass(sig1).isPresent()); From bb378802d0c843e7b1025ec39524eeafea5cd589 Mon Sep 17 00:00:00 2001 From: "M.Schmidt" Date: Wed, 22 Jan 2025 15:34:23 +0100 Subject: [PATCH 3/4] add license headers --- .../ClassFileBasedAnalysisInputLocation.java | 22 +++++++++++++++++++ .../DirectoryBasedAnalysisInputLocation.java | 22 +++++++++++++++++++ .../WarArchiveAnalysisInputLocation.java | 22 +++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java index 3e516f6def5..979d7bc33d0 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/ClassFileBasedAnalysisInputLocation.java @@ -1,5 +1,27 @@ package sootup.java.bytecode.frontend.inputlocation; +/*- + * #%L + * SootUp + * %% + * Copyright (C) 1997 - 2024 Raja Vallée-Rai and others + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 2.1 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + import java.nio.file.Files; import java.nio.file.Path; import java.util.List; diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java index 64fdf9f283c..8be9c499093 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/DirectoryBasedAnalysisInputLocation.java @@ -1,5 +1,27 @@ package sootup.java.bytecode.frontend.inputlocation; +/*- + * #%L + * Soot + * %% + * Copyright (C) 2018-2020 Manuel Benz, Christian Brüggemann, Markus Schmidt and others + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 2.1 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + import java.nio.file.Path; import java.util.Collection; import java.util.Collections; diff --git a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java index df05e9c46f3..b22b99b3d54 100644 --- a/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java +++ b/sootup.java.bytecode.frontend/src/main/java/sootup/java/bytecode/frontend/inputlocation/WarArchiveAnalysisInputLocation.java @@ -1,5 +1,27 @@ package sootup.java.bytecode.frontend.inputlocation; +/*- + * #%L + * Soot + * %% + * Copyright (C) 2018-2024 Markus Schmidt and others + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation, either version 2.1 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Lesser Public License for more details. + * + * You should have received a copy of the GNU General Lesser Public + * License along with this program. If not, see + * . + * #L% + */ + import java.io.*; import java.nio.file.Files; import java.nio.file.Path; From 2511462b65ab4c97f1266e27094dec4b51c5cc22 Mon Sep 17 00:00:00 2001 From: "M.Schmidt" Date: Wed, 22 Jan 2025 15:43:37 +0100 Subject: [PATCH 4/4] fix import --- .../interceptors/StaticSingleAssignmentFormerTest.java | 5 +++-- .../src/test/java/sootup/tests/InsertBeforeAfterTest.java | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/StaticSingleAssignmentFormerTest.java b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/StaticSingleAssignmentFormerTest.java index 15a21a8c3dc..2702f3490ec 100644 --- a/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/StaticSingleAssignmentFormerTest.java +++ b/sootup.java.bytecode.frontend/src/test/java/sootup/java/bytecode/frontend/interceptors/StaticSingleAssignmentFormerTest.java @@ -23,6 +23,7 @@ import sootup.core.types.VoidType; import sootup.core.util.ImmutableUtils; import sootup.interceptors.StaticSingleAssignmentFormer; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; import sootup.java.core.JavaIdentifierFactory; import sootup.java.core.language.JavaJimple; @@ -145,7 +146,7 @@ public void testSSA2() { clazzType, "main", "void", Collections.singletonList("java.lang.String[]")); final Path path = Paths.get(location + "TrapSSA.class"); PathBasedAnalysisInputLocation inputLocationWithSSA = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( path, "", SourceType.Application, @@ -252,7 +253,7 @@ public void testSSA3() { clazzType, "main", "void", Collections.singletonList("java.lang.String[]")); final Path path = Paths.get(location + "ForLoopSSA.class"); PathBasedAnalysisInputLocation inputLocationWithSSA = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( path, "", SourceType.Application, diff --git a/sootup.tests/src/test/java/sootup/tests/InsertBeforeAfterTest.java b/sootup.tests/src/test/java/sootup/tests/InsertBeforeAfterTest.java index 8fdba8e8e1d..23a91c63cba 100644 --- a/sootup.tests/src/test/java/sootup/tests/InsertBeforeAfterTest.java +++ b/sootup.tests/src/test/java/sootup/tests/InsertBeforeAfterTest.java @@ -17,6 +17,7 @@ import sootup.core.model.SourceType; import sootup.core.signatures.MethodSignature; import sootup.core.types.ClassType; +import sootup.java.bytecode.frontend.inputlocation.ClassFileBasedAnalysisInputLocation; import sootup.java.bytecode.frontend.inputlocation.PathBasedAnalysisInputLocation; import sootup.java.core.JavaIdentifierFactory; import sootup.java.core.language.JavaJimple; @@ -35,7 +36,7 @@ public class InsertBeforeAfterTest { factory.getMethodSignature(clazzType, "test", "void", Collections.emptyList()); final Path path = Paths.get(location + "TrapBlockCheck.class"); PathBasedAnalysisInputLocation inputLocation = - new PathBasedAnalysisInputLocation.ClassFileBasedAnalysisInputLocation( + new ClassFileBasedAnalysisInputLocation( path, "", SourceType.Application, Collections.emptyList()); JavaView view = new JavaView(inputLocation); Body body = view.getMethod(methodSignature).get().getBody();