Skip to content

Commit

Permalink
Development: Cache windfiles on startup (#7817)
Browse files Browse the repository at this point in the history
  • Loading branch information
@reschandreas
reschandreas authored Dec 29, 2023
1 parent ce8cb00 commit 25177d2
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 92 deletions.
33 changes: 31 additions & 2 deletions src/main/java/de/tum/in/www1/artemis/service/connectors/aeolus/AeolusTemplateService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
@Profile("aeolus | localci")
public class AeolusTemplateService {

private final Logger LOGGER = LoggerFactory.getLogger(AeolusTemplateService.class);
private final Logger logger = LoggerFactory.getLogger(AeolusTemplateService.class);

private final ProgrammingLanguageConfiguration programmingLanguageConfiguration;

Expand All @@ -44,6 +44,35 @@ public class AeolusTemplateService {
public AeolusTemplateService(ProgrammingLanguageConfiguration programmingLanguageConfiguration, ResourceLoaderService resourceLoaderService) {
this.programmingLanguageConfiguration = programmingLanguageConfiguration;
this.resourceLoaderService = resourceLoaderService;
// load all scripts into the cache
cacheOnBoot();
}

private void cacheOnBoot() {
var resources = this.resourceLoaderService.getResources(Path.of("templates", "aeolus"));
for (var resource : resources) {
try {
String filename = resource.getFilename();
if (filename == null) {
continue;
}
String directory = resource.getURL().getPath().split("templates/aeolus/")[1].split("/")[0];
String projectType = filename.split("_")[0].replace(".yaml", "");
Optional<ProjectType> optionalProjectType = Optional.empty();
if (!projectType.equals("default")) {
optionalProjectType = Optional.of(ProjectType.valueOf(projectType.toUpperCase()));
}
String uniqueKey = directory + "_" + filename;
byte[] fileContent = IOUtils.toByteArray(resource.getInputStream());
String script = new String(fileContent, StandardCharsets.UTF_8);
Windfile windfile = readWindfile(script);
this.addInstanceVariablesToWindfile(windfile, ProgrammingLanguage.valueOf(directory.toUpperCase()), optionalProjectType);
templateCache.put(uniqueKey, windfile);
}
catch (IOException | IllegalArgumentException e) {
logger.error("Failed to load windfile {}", resource.getFilename(), e);
}
}
}

/**
Expand Down Expand Up @@ -118,7 +147,7 @@ public Windfile getDefaultWindfileFor(ProgrammingExercise exercise) {
exercise.hasSequentialTestRuns(), exercise.isTestwiseCoverageEnabled());
}
catch (IOException e) {
LOGGER.info("No windfile for the settings of exercise {}", exercise.getId(), e);
logger.info("No windfile for the settings of exercise {}", exercise.getId(), e);
}
return null;
}
Expand Down
101 changes: 12 additions & 89 deletions src/main/java/de/tum/in/www1/artemis/service/connectors/localci/LocalCIContainerService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.CreateContainerResponse;
import com.github.dockerjava.api.command.ExecCreateCmd;
import com.github.dockerjava.api.command.ExecCreateCmdResponse;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Container;
Expand All @@ -37,7 +38,6 @@
import de.tum.in.www1.artemis.domain.BuildLogEntry;
import de.tum.in.www1.artemis.domain.ProgrammingExercise;
import de.tum.in.www1.artemis.domain.enumeration.ProgrammingLanguage;
import de.tum.in.www1.artemis.domain.enumeration.ProjectType;
import de.tum.in.www1.artemis.domain.participation.ProgrammingExerciseParticipation;
import de.tum.in.www1.artemis.exception.LocalCIException;
import de.tum.in.www1.artemis.service.connectors.aeolus.AeolusTemplateService;
Expand Down Expand Up @@ -131,7 +131,7 @@ public List<BuildLogEntry> runScriptInContainer(String containerId) {
// container's
// main process. The execution command can run concurrently with the main process. This setup with the ExecCreateCmdResponse gives us the ability to wait in code until the
// command has finished before trying to extract the results.
return executeDockerCommand(containerId, true, true, "sh", WORKING_DIRECTORY + "/script.sh");
return executeDockerCommand(containerId, true, true, false, "bash", WORKING_DIRECTORY + "/script.sh");
}

/**
Expand Down Expand Up @@ -235,7 +235,7 @@ public void populateBuildJobContainer(String buildJobContainerId, Path assignmen

if (!Objects.equals(testCheckoutPath, "")) {
addDirectory(buildJobContainerId, WORKING_DIRECTORY + "/testing-dir", true);
executeDockerCommand(buildJobContainerId, false, false, "chmod", "-R", "777", WORKING_DIRECTORY + "/testing-dir");
executeDockerCommand(buildJobContainerId, false, false, true, "chmod", "-R", "777", WORKING_DIRECTORY + "/testing-dir");
}
addAndPrepareDirectory(buildJobContainerId, testRepositoryPath, WORKING_DIRECTORY + "/testing-dir/" + testCheckoutPath);
addAndPrepareDirectory(buildJobContainerId, assignmentRepositoryPath, WORKING_DIRECTORY + "/testing-dir/" + assignmentCheckoutPath);
Expand All @@ -258,16 +258,16 @@ private void addAndPrepareDirectory(String containerId, Path repositoryPath, Str
}

private void renameDirectoryOrFile(String containerId, String oldName, String newName) {
executeDockerCommand(containerId, false, false, "mv", oldName, newName);
executeDockerCommand(containerId, false, false, true, "mv", oldName, newName);
}

private void addDirectory(String containerId, String directoryName, boolean createParentsIfNecessary) {
String[] command = createParentsIfNecessary ? new String[] { "mkdir", "-p", directoryName } : new String[] { "mkdir", directoryName };
executeDockerCommand(containerId, false, false, command);
executeDockerCommand(containerId, false, false, true, command);
}

private void convertDosFilesToUnix(String path, String containerId) {
executeDockerCommand(containerId, false, false, "sh", "-c", "find " + path + " -type f ! -path '*/.git/*' -exec sed -i 's/\\r$//' {} \\;");
executeDockerCommand(containerId, false, false, true, "sh", "-c", "find " + path + " -type f ! -path '*/.git/*' -exec sed -i 's/\\r$//' {} \\;");
}

private void copyToContainer(String sourcePath, String containerId) {
Expand Down Expand Up @@ -328,11 +328,14 @@ private void executeDockerCommandWithoutAwaitingResponse(String containerId, Str
dockerClient.execStartCmd(createCmdResponse.getId()).withDetach(true).exec(new ResultCallback.Adapter<>());
}

private List<BuildLogEntry> executeDockerCommand(String containerId, boolean attachStdout, boolean attachStderr, String... command) {
private List<BuildLogEntry> executeDockerCommand(String containerId, boolean attachStdout, boolean attachStderr, boolean forceRoot, String... command) {
boolean detach = !attachStdout && !attachStderr;

ExecCreateCmdResponse execCreateCmdResponse = dockerClient.execCreateCmd(containerId).withAttachStdout(attachStdout).withAttachStderr(attachStderr).withUser("root")
.withCmd(command).exec();
ExecCreateCmd execCreateCmd = dockerClient.execCreateCmd(containerId).withAttachStdout(attachStdout).withAttachStderr(attachStderr).withCmd(command);
if (forceRoot) {
execCreateCmd = execCreateCmd.withUser("root");
}
ExecCreateCmdResponse execCreateCmdResponse = execCreateCmd.exec();
List<BuildLogEntry> buildLogEntries = new ArrayList<>();
final CountDownLatch latch = new CountDownLatch(1);
dockerClient.execStartCmd(execCreateCmdResponse.getId()).withDetach(detach).exec(new ResultCallback.Adapter<>() {
Expand Down Expand Up @@ -369,8 +372,6 @@ public void onComplete() {
*/
public Path createBuildScript(ProgrammingExerciseParticipation participation, String containerName) {
ProgrammingExercise programmingExercise = participation.getProgrammingExercise();
boolean hasSequentialTestRuns = programmingExercise.hasSequentialTestRuns();
boolean hasStaticCodeAnalysis = programmingExercise.isStaticCodeAnalysisEnabled();

List<ScriptAction> actions = List.of();

Expand Down Expand Up @@ -412,19 +413,6 @@ public Path createBuildScript(ProgrammingExerciseParticipation participation, St
}
});

// Fall back to hardcoded scripts for old exercises without windfile
// *****************
// TODO: Delete once windfile templates can be used as fallbacks
if (actions.isEmpty()) {
// Windfile actions are not defined, use default build script
switch (programmingExercise.getProgrammingLanguage()) {
case JAVA, KOTLIN -> scriptForJavaKotlin(programmingExercise, buildScript, hasSequentialTestRuns, hasStaticCodeAnalysis);
case PYTHON -> scriptForPython(buildScript);
default -> throw new IllegalArgumentException("No build stage setup for programming language " + programmingExercise.getProgrammingLanguage());
}
}
// *****************

try {
FileUtils.writeStringToFile(buildScriptPath.toFile(), buildScript.toString(), StandardCharsets.UTF_8);
}
Expand All @@ -435,71 +423,6 @@ public Path createBuildScript(ProgrammingExerciseParticipation participation, St
return buildScriptPath;
}

private void scriptForJavaKotlin(ProgrammingExercise programmingExercise, StringBuilder buildScript, boolean hasSequentialTestRuns, boolean hasStaticCodeAnalysis) {
boolean isMaven = ProjectType.isMavenProject(programmingExercise.getProjectType());

if (isMaven) {
if (hasSequentialTestRuns) {
buildScript.append("""
cd structural
mvn clean test
if [ $? -eq 0 ]; then
cd ..
cd behavior
mvn clean test
fi
cd ..
""");
}
else {
buildScript.append("""
mvn clean test
""");
if (hasStaticCodeAnalysis) {
buildScript.append("""
mvn checkstyle:checkstyle
mvn pmd:pmd
mvn spotbugs:spotbugs
""");
}
}
}
else {
if (hasSequentialTestRuns) {
buildScript.append("""
chmod +x gradlew
./gradlew clean structuralTests
if [ $? -eq 0 ]; then
./gradlew behaviorTests
fi
""");
}
else {
buildScript.append("""
chmod +x gradlew
./gradlew clean test
""");
if (hasStaticCodeAnalysis) {
buildScript.append("""
./gradlew check -x test
""");
}
}
}
}

private void scriptForPython(StringBuilder buildScript) {
buildScript.append("""
python3 -m compileall . -q || error=true
if [ ! $error ]
then
pytest --junitxml=test-reports/results.xml
else
exit 1
fi
""");
}

/**
* Deletes the build script for a given programming exercise.
* The build script is stored in a file in the local-ci-scripts directory.
Expand Down
2 changes: 1 addition & 1 deletion .../in/www1/artemis/service/connectors/localci/LocalCIProgrammingLanguageFeatureService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public LocalCIProgrammingLanguageFeatureService() {
programmingLanguageFeatures.put(PYTHON, new ProgrammingLanguageFeature(PYTHON, false, false, true, false, false, List.of(), false, false, true));
programmingLanguageFeatures.put(C, new ProgrammingLanguageFeature(C, false, true, true, false, false, List.of(FACT, GCC), false, true, true));
programmingLanguageFeatures.put(ASSEMBLER, new ProgrammingLanguageFeature(ASSEMBLER, false, false, false, false, false, List.of(), false, true, true));
programmingLanguageFeatures.put(KOTLIN, new ProgrammingLanguageFeature(KOTLIN, true, false, true, true, false, List.of(), false, true, true));
programmingLanguageFeatures.put(KOTLIN, new ProgrammingLanguageFeature(KOTLIN, false, false, true, true, false, List.of(), false, true, true));
programmingLanguageFeatures.put(VHDL, new ProgrammingLanguageFeature(VHDL, false, false, false, false, false, List.of(), false, true, true));
programmingLanguageFeatures.put(HASKELL, new ProgrammingLanguageFeature(HASKELL, true, false, false, false, true, List.of(), false, true, true));
programmingLanguageFeatures.put(OCAML, new ProgrammingLanguageFeature(OCAML, false, false, false, false, true, List.of(), false, true, true));
Expand Down

0 comments on commit 25177d2

Please sign in to comment.