Adjustments in Last Merge experiment

src/main/app/MiningWorker.groovy

@@ -1,27 +1,37 @@
 package app
 
+import exception.UnstagedChangesException
+import interfaces.CommitFilter
+import interfaces.DataCollector
+import org.apache.logging.log4j.LogManager
+import org.apache.logging.log4j.Logger
+import project.MergeCommit
+import project.Project
+import project.commitHashesExtraction.CommitHashesExtractor
+import services.util.Utils
+import util.FileManager
+import util.MergeHelper
+import util.ProcessRunner
 import java.util.Collections
 import java.util.Random
 import java.text.SimpleDateFormat
 
-import static app.MiningFramework.arguments
+import java.text.SimpleDateFormat
 import java.util.concurrent.BlockingQueue
 
-import project.*
-import interfaces.*
-import exception.UnstagedChangesException
-import util.*
-
-import services.util.Utils;
+import static app.MiningFramework.arguments
 
 class MiningWorker implements Runnable {
+    private static Logger LOG = LogManager.getLogger(MiningWorker.class)
 
     private Set<DataCollector> dataCollectors
     private CommitFilter commitFilter
     private BlockingQueue<Project> projectList
     private String baseDir
+    private CommitHashesExtractor commitHashesExtractor
 
     MiningWorker(Set<DataCollector> dataCollectors, CommitFilter commitFilter, BlockingQueue<Project> projectList, String baseDir) {
+        this.commitHashesExtractor = CommitHashesExtractor.Factory.build()
         this.dataCollectors = dataCollectors
         this.commitFilter = commitFilter
         this.projectList = projectList
@@ -41,7 +51,7 @@ class MiningWorker implements Runnable {
                     checkForUnstagedChanges(project);
                 }
 
-                def (mergeCommits, skipped) = project.getMergeCommits(arguments.getSinceDate(), arguments.getUntilDate())
+                def (mergeCommits, skipped) = project.getMergeCommits(commitHashesExtractor)
 
                 Random random = new Random(arguments.getRandomSeed())
                 Collections.shuffle(mergeCommits, random)
@@ -97,7 +107,7 @@ class MiningWorker implements Runnable {
     }
 
     private void cloneRepository(Project project, String target) {
-        println "Cloning repository ${project.getName()} into ${target}"
+        LOG.info("Cloning repository ${project.getName()} into ${target}")
 
         File projectDirectory = new File(target)
         if (projectDirectory.exists()) {
@@ -114,10 +124,11 @@ class MiningWorker implements Runnable {
         }
 
         ProcessBuilder builder = ProcessRunner.buildProcess('./', 'git', 'clone', url, target)
-        builder.redirectOutput(ProcessBuilder.Redirect.INHERIT)
-
         Process process = ProcessRunner.startProcess(builder)
+        process.getInputStream().eachLine(LOG::trace)
+        process.getErrorStream().eachLine(LOG::warn)
         process.waitFor()
+        LOG.info("Finished cloning repository ${project.getName()} into ${target}")
 
         project.setPath(target)
     }

src/main/arguments/ArgsParser.groovy

@@ -39,6 +39,7 @@ class ArgsParser {
         this.cli.e(longOpt: 'extension', args: 1, argName: 'file extenson', 'Specify the file extension that should be used in the analysis (e.g. .rb, .ts, .java, .cpp. Default: .java)')
         this.cli.l(longOpt: 'language-separators', args: 1, argName: 'language syntactic separators', 'Specify the language separators that should be used in the analysis. Required for (and only considered when) running studies with the CSDiff tool. Default: \"{ } ( ) ; ,\"')
         this.cli.log(longOpt: 'log-level', args: 1, argName: 'log level', 'Specify the minimum log level: (OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE, ALL). Default: \"INFO\"')
+        this.cli.pchf(longOpt: 'project-commit-hashes-file', args: 1, argName: 'project commit hashes file', 'Path to a file with projects and commits to be analyzed')
     }
 
     Arguments parse(args) {
@@ -142,6 +143,10 @@ class ArgsParser {
         if(this.options.log) {
             args.setLogLevel(Level.toLevel(this.options.log))
         }
+
+        if (this.options.pchf) {
+            args.setProjectCommitHashesFile(this.options.pchf)
+        }
     }
 
     private boolean repositoryExists(String repositoryURL) {

src/main/arguments/Arguments.groovy

@@ -21,6 +21,7 @@ class Arguments {
     private String syntacticSeparators
     private String fileExtension
     private Level logLevel
+    private String projectCommitHashesFile
 
     Arguments() { // set the default values for all parameters
         randomSeed = 1
@@ -37,6 +38,7 @@ class Arguments {
         syntacticSeparators = '{ } ( ) ; ,'
         fileExtension = '.java'
         logLevel = Level.INFO
+        projectCommitHashesFile = ''
     }
 
     void setRandomSeed(int randomSeed) {
@@ -167,4 +169,12 @@ class Arguments {
         this.logLevel = logLevel
         Configurator.setRootLevel(logLevel)
     }
+
+    String getProjectCommitHashesFile() {
+        return projectCommitHashesFile
+    }
+
+    void setProjectCommitHashesFile(String projectCommitHashesFile) {
+        this.projectCommitHashesFile = projectCommitHashesFile
+    }
 }

src/main/injectors/GenericMergeModule.groovy

@@ -6,51 +6,77 @@ import interfaces.CommitFilter
 import interfaces.DataCollector
 import interfaces.OutputProcessor
 import interfaces.ProjectProcessor
-import org.apache.logging.log4j.LogManager
-import org.apache.logging.log4j.Logger
 import services.commitFilters.MutuallyModifiedFilesCommitFilter
-import services.dataCollectors.GenericMerge.GenericMergeConfig
-import services.dataCollectors.GenericMerge.GenericMergeDataCollector
-import services.dataCollectors.GenericMerge.MergeConflictsComparator
-import services.dataCollectors.GenericMerge.MergeToolsComparator
-import services.dataCollectors.GenericMerge.ScenarioLOCsCounter
-import services.dataCollectors.GenericMerge.UnstructuredMergeCollector
-import services.outputProcessors.GenericMergeDataOutputProcessor
+import services.dataCollectors.buildRequester.RequestBuildForRevisionWithFilesDataCollector
+import services.dataCollectors.common.CompareScenarioMergeConflictsDataCollector
+import services.dataCollectors.common.RunDataCollectorsInParallel
+import services.dataCollectors.common.SyntacticallyCompareScenarioFilesDataCollector
+import services.dataCollectors.fileSyntacticNormalization.FormatFileSyntacticNormalizationDataCollector
+import services.dataCollectors.fileSyntacticNormalization.JDimeFileSyntacticNormalizationDataCollector
+import services.dataCollectors.fileSyntacticNormalization.SporkFileSyntacticNormalizationDataCollector
+import services.dataCollectors.mergeToolExecutors.JDimeMergeToolExecutorDataCollector
+import services.dataCollectors.mergeToolExecutors.LastMergeMergeToolExecutorDataCollector
+import services.dataCollectors.mergeToolExecutors.MergirafMergeToolExecutorDataCollector
+import services.dataCollectors.mergeToolExecutors.SporkMergeToolExecutorDataCollector
+import services.outputProcessors.EmptyOutputProcessor
 import services.projectProcessors.DummyProjectProcessor
-import services.util.ci.CIPlatform
-import services.util.ci.TravisPlatform
-
-import java.nio.file.Files
-import java.nio.file.Paths
 
 class GenericMergeModule extends AbstractModule {
-    private static Logger LOG = LogManager.getLogger(GenericMergeModule.class)
-
     @Override
     protected void configure() {
         Multibinder<ProjectProcessor> projectProcessorBinder = Multibinder.newSetBinder(binder(), ProjectProcessor.class)
         projectProcessorBinder.addBinding().to(DummyProjectProcessor.class)
 
         Multibinder<DataCollector> dataCollectorBinder = Multibinder.newSetBinder(binder(), DataCollector.class)
-        dataCollectorBinder.addBinding().to(ScenarioLOCsCounter.class)
-        dataCollectorBinder.addBinding().to(GenericMergeDataCollector.class)
-        dataCollectorBinder.addBinding().to(MergeToolsComparator.class)
-        dataCollectorBinder.addBinding().to(MergeConflictsComparator.class)
-        dataCollectorBinder.addBinding().to(UnstructuredMergeCollector.class)
 
-        Multibinder<OutputProcessor> outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class)
-        outputProcessorBinder.addBinding().to(GenericMergeDataOutputProcessor.class)
+        // Run the merge tools on the scenarios
+        dataCollectorBinder.addBinding().to(JDimeMergeToolExecutorDataCollector.class)
+        dataCollectorBinder.addBinding().to(LastMergeMergeToolExecutorDataCollector.class)
+        dataCollectorBinder.addBinding().to(SporkMergeToolExecutorDataCollector.class)
+        dataCollectorBinder.addBinding().to(MergirafMergeToolExecutorDataCollector.class)
 
-        bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class)
-        bind(CIPlatform.class).to(TravisPlatform.class)
+        // Run, in parallel, normalizations on resulting files
+        dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([
+                new JDimeFileSyntacticNormalizationDataCollector("merge.java", "merge.jdime_normalized.java"),
+                new JDimeFileSyntacticNormalizationDataCollector("merge.last_merge.java", "merge.last_merge.jdime_normalized.java"),
+                new SporkFileSyntacticNormalizationDataCollector("merge.java", "merge.spork_normalized.java"),
+                new SporkFileSyntacticNormalizationDataCollector("merge.spork.java", "merge.spork.spork_normalized.java"),
+                new SporkFileSyntacticNormalizationDataCollector("merge.mergiraf.java", "merge.mergiraf.spork_normalized.java"),
+        ]))
 
-        createExecutionReportsFile()
-    }
+        // Run, in parallel, syntactical comparisons between files and textual comparison between commits
+        dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([
+                // Syntactic comparison with merge commits
+                new SyntacticallyCompareScenarioFilesDataCollector("merge.java", "merge.last_merge.java"),
+                new SyntacticallyCompareScenarioFilesDataCollector("merge.jdime_normalized.java", "merge.jdime.java"),
+                new SyntacticallyCompareScenarioFilesDataCollector("merge.spork_normalized.java", "merge.spork.spork_normalized.java"),
+                new SyntacticallyCompareScenarioFilesDataCollector("merge.java", "merge.mergiraf.java"),
+
+                // Syntactic comparison between tools themselves
+                new SyntacticallyCompareScenarioFilesDataCollector("merge.jdime.java", "merge.last_merge.jdime_normalized.java"),
+                new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.spork_normalized.java", "merge.spork.spork_normalized.java"),
+                new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.java", "merge.last_merge.java"),
+
+                // Conflicts comparison between tools themselves
+                new CompareScenarioMergeConflictsDataCollector("merge.jdime.java", "merge.last_merge.java"),
+                new CompareScenarioMergeConflictsDataCollector("merge.mergiraf.java", "merge.spork.java"),
+                new CompareScenarioMergeConflictsDataCollector("merge.mergiraf.java", "merge.last_merge.java"),
+        ]))
 
-    private static void createExecutionReportsFile() {
-        LOG.info("Creating Generic Merge report file")
-        Files.createDirectories(Paths.get(GenericMergeConfig.GENERIC_MERGE_REPORT_PATH))
-        def reportFile = new File(GenericMergeConfig.GENERIC_MERGE_REPORT_FILE_NAME)
-        reportFile.createNewFile()
+        dataCollectorBinder.addBinding().toInstance(new RunDataCollectorsInParallel([new FormatFileSyntacticNormalizationDataCollector("merge.last_merge.java", "merge.last_merge.format_normalized.java"),
+                                                                                     new FormatFileSyntacticNormalizationDataCollector("merge.mergiraf.java", "merge.mergiraf.format_normalized.java")]))
+
+        dataCollectorBinder.addBinding().toInstance(new SyntacticallyCompareScenarioFilesDataCollector("merge.mergiraf.format_normalized.java", "merge.last_merge.format_normalized.java"))
+
+
+        dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.last_merge.java"))
+        dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.jdime.java"))
+        dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.mergiraf.java"))
+        dataCollectorBinder.addBinding().toInstance(new RequestBuildForRevisionWithFilesDataCollector("merge.spork.java"))
+
+        Multibinder<OutputProcessor> outputProcessorBinder = Multibinder.newSetBinder(binder(), OutputProcessor.class)
+        outputProcessorBinder.addBinding().to(EmptyOutputProcessor.class)
+
+        bind(CommitFilter.class).to(MutuallyModifiedFilesCommitFilter.class)
     }
 }

src/main/project/Project.groovy

@@ -1,14 +1,15 @@
 package project
 
-import util.ProcessRunner
 import exception.UnexpectedOutputException
+import org.apache.logging.log4j.LogManager
+import project.commitHashesExtraction.CommitHashesExtractor
+import util.ProcessRunner
 
-import java.util.Collections
-import java.util.Random
-import java.util.regex.Pattern
 import java.util.regex.Matcher
+import java.util.regex.Pattern
 
 class Project {
+    private static LOG = LogManager.getLogger(Project.class)
 
     private String name
     private String path
@@ -42,49 +43,41 @@ class Project {
         return matcher.find()
     }
 
-    List getMergeCommits(String sinceDate, String untilDate) {
+    List getMergeCommits(CommitHashesExtractor commitHashesExtractor) {
         ArrayList<String> skipped = new ArrayList<String>()
         ArrayList<MergeCommit> mergeCommits = new ArrayList<MergeCommit>()
-
-        Process gitLog = constructAndRunGitLog(sinceDate, untilDate)
-        def expectedOutput = ~/.*-(.* .*)+/
-        gitLog.getInputStream().eachLine {
 
-            // Each line contains the hash of the commit followed by the hashes of the parents.
-            if(it ==~ expectedOutput) {
-
-                String[] informations = it.split('-') // <commit hash>-<parents hash>
-                String SHA = getSHA(informations)
-                String[] parentsSHA = getParentsSHA(informations)
+        commitHashesExtractor.extractCommitHashes(this).each(commitHashes -> {
+            def SHA = commitHashes.mergeSha
+            def parentsSHA = commitHashes.parents
 
+            try {
+                String ancestorSHA = getCommonAncestor(SHA, parentsSHA)
+                MergeCommit mergeCommit = new MergeCommit(SHA, parentsSHA, ancestorSHA)
+                mergeCommits.add(mergeCommit)
+            } catch (UnexpectedOutputException e) {
+                println e.message
                 try {
+                    println "Trying to fetch it"
+                    // Let's try fetching it before skipping it
+                    constructAndRunGitFetch(SHA)
                     String ancestorSHA = getCommonAncestor(SHA, parentsSHA)
                     MergeCommit mergeCommit = new MergeCommit(SHA, parentsSHA, ancestorSHA)
                     mergeCommits.add(mergeCommit)
-                } catch (UnexpectedOutputException e) {
+                } catch (UnexpectedOutputException ex) {
                     println "Skipping merge commit ${SHA}"
-                    println e.message
+                    println ex.message
                     skipped.add(SHA)
                 }
-            } else {
-                throw new UnexpectedOutputException('Git log returned an unexpected output. Could not retrieve merge commits.', '<commit hash>-<parents hash>', it)
             }
-        }
-        
+        })
+
         if(mergeCommits.isEmpty())
             println "No merge commits."
 
         return [mergeCommits, skipped]
     }
 
-    private String getSHA(String[] informations) {
-        return informations[0]
-    } 
-
-    private String[] getParentsSHA(String[] informations) {
-        return informations[1].split(' ')
-    }
-
     private String getCommonAncestor(mergeCommitSHA, parentsSHA) {
         Process gitMergeBase = constructAndRunGitMergeBase(mergeCommitSHA, parentsSHA)
         def expectedOutput = ~/[0-9a-z]{7,}/
@@ -96,6 +89,14 @@ class Project {
             throw new UnexpectedOutputException('Git merge-base returned an unexpected output. Could not retrieve the ancestor commit.', '<commit hash>', actualOutput)
     }
 
+    private void constructAndRunGitFetch(String mergeCommitSHA) {
+        ProcessBuilder gitMergeBaseBuilder = ProcessRunner.buildProcess(path, 'git', 'fetch', 'origin', mergeCommitSHA)
+        def exitCode = ProcessRunner.startProcess(gitMergeBaseBuilder).waitFor()
+        if (exitCode > 0) {
+            throw new UnexpectedOutputException("Could not fetch merge commit ${mergeCommitSHA} from origin", "Exit code 0", "Exit code ${exitCode}")
+        }
+    }
+
     private Process constructAndRunGitMergeBase(String mergeCommitSHA, String[] parentsSHA) {
         ProcessBuilder gitMergeBaseBuilder = ProcessRunner.buildProcess(path, 'git', 'merge-base')
         if (parentsSHA.length > 2)
@@ -105,15 +106,6 @@ class Project {
         return ProcessRunner.startProcess(gitMergeBaseBuilder)
     }
 
-    private Process constructAndRunGitLog(String sinceDate, String untilDate) {
-        ProcessBuilder gitLogBuilder = ProcessRunner.buildProcess(path, 'git', '--no-pager', 'log', '--merges', '--pretty=%H-%p')
-        if(!sinceDate.equals(''))
-            ProcessRunner.addCommand(gitLogBuilder, "--since=\"${sinceDate}\"")
-        if(!untilDate.equals(''))
-            ProcessRunner.addCommand(gitLogBuilder, "--until=\"${untilDate}\"")
-        return ProcessRunner.startProcess(gitLogBuilder)
-    }
-
     String getName() {
         return name
     }

src/main/project/commitHashesExtraction/CommitHashesExtractor.groovy

@@ -0,0 +1,33 @@
+package project.commitHashesExtraction
+
+import project.Project
+
+import static app.MiningFramework.arguments
+
+interface CommitHashesExtractor {
+    class CommitHashes {
+        public String mergeSha
+        public String[] parents
+
+        CommitHashes(String mergeSha, String... parents) {
+            this.mergeSha = mergeSha
+            this.parents = parents
+        }
+    }
+
+    class Factory {
+        static CommitHashesExtractor build() {
+            if (arguments.projectCommitHashesFile != '') {
+                def file = new File(arguments.projectCommitHashesFile)
+                if (!file.exists()) {
+                    throw new RuntimeException("File ${file.path} not found")
+                }
+                return new ProjectCommitListCommitHashesExtractor(file)
+            }
+
+            return new GitLogCommitHashesExtractor(arguments.sinceDate, arguments.untilDate)
+        }
+    }
+
+    List<CommitHashes> extractCommitHashes(Project project)
+}