diff --git a/.idea/ClojureProjectResolveSettings.xml b/.idea/ClojureProjectResolveSettings.xml
new file mode 100644
index 00000000..df470b16
--- /dev/null
+++ b/.idea/ClojureProjectResolveSettings.xml
@@ -0,0 +1,6 @@
+
+
+
+ IDE
+
+
\ No newline at end of file
diff --git a/.idea/CodeMetropolis-sz10.iml b/.idea/CodeMetropolis-sz10.iml
new file mode 100644
index 00000000..d6ebd480
--- /dev/null
+++ b/.idea/CodeMetropolis-sz10.iml
@@ -0,0 +1,9 @@
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml
new file mode 100644
index 00000000..919ce1f1
--- /dev/null
+++ b/.idea/codeStyles/Project.xml
@@ -0,0 +1,7 @@
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml
new file mode 100644
index 00000000..a55e7a17
--- /dev/null
+++ b/.idea/codeStyles/codeStyleConfig.xml
@@ -0,0 +1,5 @@
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 00000000..aa345419
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 00000000..dd03e277
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
new file mode 100644
index 00000000..35eb1ddf
--- /dev/null
+++ b/.idea/vcs.xml
@@ -0,0 +1,6 @@
+
+
+
+
+
+
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
new file mode 100644
index 00000000..6a0cbfe5
--- /dev/null
+++ b/.idea/workspace.xml
@@ -0,0 +1,84 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1647969322929
+
+
+ 1647969322929
+
+
+
+
\ No newline at end of file
diff --git a/demo-sz10 b/demo-sz10
new file mode 100644
index 00000000..e69de29b
diff --git a/sources/toolchain/converter/src/main/java/codemetropolis/toolchain/converter/sonarqube/SonarQubeConverter.java b/sources/toolchain/converter/src/main/java/codemetropolis/toolchain/converter/sonarqube/SonarQubeConverter.java
index f3199189..57b88386 100644
--- a/sources/toolchain/converter/src/main/java/codemetropolis/toolchain/converter/sonarqube/SonarQubeConverter.java
+++ b/sources/toolchain/converter/src/main/java/codemetropolis/toolchain/converter/sonarqube/SonarQubeConverter.java
@@ -1,11 +1,14 @@
package codemetropolis.toolchain.converter.sonarqube;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import codemetropolis.toolchain.commons.cdf.CdfElement;
import codemetropolis.toolchain.commons.cdf.CdfProperty.Type;
@@ -33,11 +36,17 @@ public SonarQubeConverter(Map params) {
resources = new HashMap<>();
cdfElements = new HashMap<>();
}
-
+/**
+ * Method for getting the projects and turning them into cdfelements, returning a tree
+ * containing them all.
+ * */
@Override
public CdfTree createElements(String url) throws CodeMetropolisException {
resources.putAll(getResources(url));
-
+
+ List projectResources = new ArrayList<>();
+ List processedProjects;
+
CdfTree cdfTree = new CdfTree();
Iterator iterator = resources.keySet().iterator();
@@ -48,11 +57,24 @@ public CdfTree createElements(String url) throws CodeMetropolisException {
Integer rootId = iterator.next();
SonarResource res = resources.get(rootId);
if(Scope.PRJ.equals(res.getScope())){
- CdfElement projectElement = createCdfElement(res);
- rootElement.addChildElement(projectElement);
+ projectResources.add(res);
}
}
-
+ Stream resourceStream = projectResources.stream();
+ if(projectResources.size() > (Runtime.getRuntime().availableProcessors()) * 10){ // not a computationally intensive process, parallelization only provides an advantage when a large number of projects is being processed
+ resourceStream = resourceStream.parallel();
+ }
+ processedProjects = resourceStream.map(this::createCdfElement).collect(Collectors.toList());
+ // in some cases, a SonarResource in projectResources may have a null element in childIdList causing the above line to fail. This is (most likely) caused by parallel downloading resulting in faulty data. Rerunning the tool usually fixes this issue
+
+ for (CdfElement projectElement : processedProjects) {
+ rootElement.addChildElement(projectElement);
+ }
+ System.gc();
+ // when ran in parallel, threads sometimes free up slowly, making the next step in the converting process slow
+ // (in some cases making it take longer to finish printing the result to file, negating the advantage won with parallel processing the projects)
+ // calling gc fixes this issue in most cases
+
String splitDirsParam = getParameter(SPLIT_DIRS_PARAM_NAME);
if(splitDirsParam != null && Boolean.valueOf(splitDirsParam)) {
processDirHierarchy(cdfTree);
@@ -68,32 +90,42 @@ private String[] getProjectsInParams() {
}
private Map getResources(String url) throws SonarConnectException {
- Map result = new HashMap<>();
-
+ Map result;
+
SonarClient sonarClient = new SonarClient(url, getParameter(USERNAME_PARAM_NAME), getParameter(PASSWORD_PARAM_NAME));
sonarClient.init();
String[] projectRegexList = getProjectsInParams();
List allProjects = sonarClient.getProjectKeys();
- Set projects = new LinkedHashSet<>();
-
+ Set projects;
+
if(projectRegexList.length == 0) {
- projects.addAll(allProjects);
+ projects = new LinkedHashSet<>(allProjects);
} else {
- for(String p : allProjects) {
- for(String regex : projectRegexList) {
- if(p.matches(regex)) {
- projects.add(p);
- break;
- }
+ projects = allProjects.stream().filter(p -> {
+ for (String regex : projectRegexList) {
+ if(p.matches(regex))
+ return true;
}
- }
+ return false;
+ }).collect(Collectors.toSet());
}
-
- for(String key : projects) {
- fireConverterEvent(String.format(Resources.get("sonar_downloading_project"), key));
- result.putAll(sonarClient.getProject(key));
+
+ Stream projectStream = projects.stream();
+ if(projects.size() > (Runtime.getRuntime().availableProcessors() * 2)){
+ // usefulness of parallelization heavily depends on project size, which is unknown at this point.
+ projectStream = projectStream.parallel();
}
+ result = projectStream
+ .peek(projectKey -> fireConverterEvent(String.format(Resources.get("sonar_downloading_project"), projectKey)))
+ .map(projectKey -> {
+ try {
+ return sonarClient.getProject(projectKey);
+ } catch (SonarConnectException e) {
+ throw new RuntimeException(e);
+ }
+ })
+ .collect(HashMap::new, HashMap::putAll, Map::putAll);
return result;
}