Enable loading model with references from GitHub

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/RdfUtil.java

@@ -36,6 +36,7 @@
 import org.apache.jena.rdf.model.ModelFactory;
 import org.apache.jena.rdf.model.Property;
 import org.apache.jena.rdf.model.RDFNode;
+import org.apache.jena.rdf.model.ResIterator;
 import org.apache.jena.rdf.model.Resource;
 import org.apache.jena.rdf.model.Statement;
 import org.apache.jena.vocabulary.RDF;
@@ -165,4 +166,32 @@ public static List<Resource> getNamedElementsReferringTo( final RDFNode node ) {
          } ).toList();
       }
    }
+
+   /**
+    * Merges an RDF-model into another on a per-element basis instead of a per-statement basis. This means only those model element
+    * definitions from the model to merge are merged into the target model that are not already present in the target model.
+    * This prevents duplicate assertions of statements where the object is a blank node.
+    *
+    * @param target the model to merge into
+    * @param modelToMerge the source model of model element definitions to merge
+    */
+   public static void mergeModel( final Model target, final Model modelToMerge ) {
+      final Set<String> targetSubjects = Streams.stream( target.listSubjects() )
+            .filter( Resource::isURIResource )
+            .map( Resource::getURI )
+            .collect( toSet() );
+      for ( final ResIterator it = modelToMerge.listSubjects(); it.hasNext(); ) {
+         final Resource resource = it.next();
+         if ( resource.isAnon() ) {
+            continue;
+         }
+         if ( !targetSubjects.contains( resource.getURI() ) ) {
+            final Model modelElementDefinition = getModelElementDefinition( resource );
+            target.add( modelElementDefinition );
+         }
+      }
+      for ( final Map.Entry<String, String> prefixEntry : modelToMerge.getNsPrefixMap().entrySet() ) {
+         target.setNsPrefix( prefixEntry.getKey(), prefixEntry.getValue() );
+      }
+   }
 }

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/loader/AspectModelLoader.java

@@ -43,18 +43,20 @@
 import org.eclipse.esmf.aspectmodel.resolver.AspectModelFileLoader;
 import org.eclipse.esmf.aspectmodel.resolver.EitherStrategy;
 import org.eclipse.esmf.aspectmodel.resolver.FileSystemStrategy;
-import org.eclipse.esmf.aspectmodel.resolver.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.resolver.ModelSource;
 import org.eclipse.esmf.aspectmodel.resolver.ResolutionStrategy;
 import org.eclipse.esmf.aspectmodel.resolver.ResolutionStrategySupport;
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.resolver.fs.FlatModelsRoot;
 import org.eclipse.esmf.aspectmodel.resolver.modelfile.DefaultAspectModelFile;
 import org.eclipse.esmf.aspectmodel.resolver.modelfile.MetaModelFile;
 import org.eclipse.esmf.aspectmodel.resolver.modelfile.RawAspectModelFile;
+import org.eclipse.esmf.aspectmodel.resolver.services.TurtleLoader;
 import org.eclipse.esmf.aspectmodel.urn.AspectModelUrn;
 import org.eclipse.esmf.aspectmodel.urn.ElementType;
 import org.eclipse.esmf.aspectmodel.urn.UrnSyntaxException;
 import org.eclipse.esmf.aspectmodel.versionupdate.MetaModelVersionMigrator;
+import org.eclipse.esmf.metamodel.Aspect;
 import org.eclipse.esmf.metamodel.AspectModel;
 import org.eclipse.esmf.metamodel.ModelElement;
 import org.eclipse.esmf.metamodel.Namespace;
@@ -113,6 +115,7 @@ public AspectModelLoader( final ResolutionStrategy resolutionStrategy ) {
     * @param resolutionStrategies the strategies
     */
    public AspectModelLoader( final List<ResolutionStrategy> resolutionStrategies ) {
+      TurtleLoader.init();
       if ( resolutionStrategies.size() == 1 ) {
          resolutionStrategy = resolutionStrategies.get( 0 );
       } else if ( resolutionStrategies.isEmpty() ) {
@@ -434,10 +437,10 @@ public AspectModel emptyModel() {
     */
    public AspectModel loadAspectModelFiles( final Collection<AspectModelFile> inputFiles ) {
       final Model mergedModel = ModelFactory.createDefaultModel();
-      mergedModel.add( MetaModelFile.metaModelDefinitions() );
       for ( final AspectModelFile file : inputFiles ) {
-         mergedModel.add( file.sourceModel() );
+         RdfUtil.mergeModel( mergedModel, file.sourceModel() );
       }
+      mergedModel.add( MetaModelFile.metaModelDefinitions() );
 
       final List<ModelElement> elements = new ArrayList<>();
       final List<AspectModelFile> files = new ArrayList<>();
@@ -460,6 +463,10 @@ public AspectModel loadAspectModelFiles( final Collection<AspectModelFile> input
       }
 
       setNamespaces( files, elements );
+      elements.stream()
+            .filter( modelElement -> modelElement.is( Aspect.class ) )
+            .findFirst()
+            .ifPresent( aspect -> mergedModel.setNsPrefix( "", aspect.urn().getUrnPrefix() ) );
       return new DefaultAspectModel( files, mergedModel, elements );
    }
 

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/AspectModelFileLoader.java

@@ -16,6 +16,7 @@
 import static org.apache.commons.lang3.StringUtils.isBlank;
 
 import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -34,6 +35,7 @@
 
 import org.eclipse.esmf.aspectmodel.AspectModelFile;
 import org.eclipse.esmf.aspectmodel.loader.AspectModelLoader;
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.resolver.exceptions.ParserException;
 import org.eclipse.esmf.aspectmodel.resolver.modelfile.RawAspectModelFile;
 import org.eclipse.esmf.aspectmodel.resolver.services.TurtleLoader;
@@ -96,15 +98,24 @@ public static RawAspectModelFile load( final Model model ) {
       return new RawAspectModelFile( model, List.of(), Optional.empty() );
    }
 
+   public static RawAspectModelFile load( final byte[] content ) {
+      return load( new ByteArrayInputStream( content ) );
+   }
+
    public static RawAspectModelFile load( final URL url ) {
       if ( url.getProtocol().equals( "file" ) ) {
          try {
             return load( Paths.get( url.toURI() ).toFile() );
          } catch ( final URISyntaxException exception ) {
             throw new ModelResolutionException( "Can not load model from file URL", exception );
          }
+      } else if ( url.getProtocol().equals( "http" ) || url.getProtocol().equals( "https" ) ) {
+         // Downloading from http(s) should take proxy settings into consideration, so we don't just .openStream() here
+         final byte[] fileContent = new Download().downloadFile( url );
+         return load( fileContent );
       }
       try {
+         // Other URLs (e.g. resource://) we just load using openStream()
          return load( url.openStream(), Optional.of( url.toURI() ) );
       } catch ( final IOException | URISyntaxException exception ) {
          throw new ModelResolutionException( "Can not load model from URL", exception );

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/ClasspathStrategy.java

@@ -33,6 +33,7 @@
 import java.util.stream.Stream;
 
 import org.eclipse.esmf.aspectmodel.AspectModelFile;
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.urn.AspectModelUrn;
 
 import org.apache.commons.io.IOUtils;

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/CommandExecutor.java

@@ -18,6 +18,8 @@
 import java.util.Scanner;
 import java.util.StringTokenizer;
 
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
+
 /**
  * Executes an external resolver via the underlying OS command and returns the stdout from the command as result.
  */

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/Download.java

@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
+ *
+ * See the AUTHORS file(s) distributed with this work for additional
+ * information regarding authorship.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package org.eclipse.esmf.aspectmodel.resolver;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+import java.util.Optional;
+
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Convenience class to download a file via HTTP, which the ability to auto-detect and use proxy settings
+ */
+public class Download {
+   private static final Logger LOG = LoggerFactory.getLogger( Download.class );
+   private final ProxyConfig proxyConfig;
+
+   public Download( final ProxyConfig proxyConfig ) {
+      this.proxyConfig = proxyConfig;
+   }
+
+   public Download() {
+      this( ProxyConfig.detectProxySettings() );
+   }
+
+   /**
+    * Download the file and return the contents as byte array
+    *
+    * @param fileUrl the URL
+    * @return the file contents
+    */
+   public byte[] downloadFile( final URL fileUrl ) {
+      try {
+         final HttpClient.Builder clientBuilder = HttpClient.newBuilder()
+               .version( HttpClient.Version.HTTP_1_1 )
+               .followRedirects( HttpClient.Redirect.ALWAYS )
+               .connectTimeout( Duration.ofSeconds( 10 ) );
+         Optional.ofNullable( proxyConfig.proxy() ).ifPresent( clientBuilder::proxy );
+         Optional.ofNullable( proxyConfig.authenticator() ).ifPresent( clientBuilder::authenticator );
+         final HttpClient client = clientBuilder.build();
+         final HttpRequest request = HttpRequest.newBuilder().uri( fileUrl.toURI() ).build();
+         final HttpResponse<byte[]> response = client.send( request, HttpResponse.BodyHandlers.ofByteArray() );
+         return response.body();
+      } catch ( final InterruptedException | URISyntaxException | IOException exception ) {
+         throw new ModelResolutionException( "Could not retrieve " + fileUrl, exception );
+      }
+   }
+
+   /**
+    * Download the file and write it to the file system
+    *
+    * @param fileUrl the URL
+    * @param outputFile the output file to write
+    * @return the file written
+    */
+   public File downloadFile( final URL fileUrl, final File outputFile ) {
+      try ( final FileOutputStream outputStream = new FileOutputStream( outputFile ) ) {
+         final byte[] fileContent = downloadFile( fileUrl );
+         outputStream.write( fileContent );
+      } catch ( final IOException exception ) {
+         throw new ModelResolutionException( "Could not write file " + outputFile, exception );
+      }
+
+      LOG.info( "Downloaded {} to local file {}", fileUrl.getPath(), outputFile );
+      return outputFile;
+   }
+}

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/EitherStrategy.java

@@ -20,6 +20,7 @@
 import java.util.stream.Stream;
 
 import org.eclipse.esmf.aspectmodel.AspectModelFile;
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.urn.AspectModelUrn;
 
 import io.vavr.control.Try;

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/FileSystemStrategy.java

@@ -22,6 +22,7 @@
 import java.util.stream.Stream;
 
 import org.eclipse.esmf.aspectmodel.AspectModelFile;
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.resolver.fs.ModelsRoot;
 import org.eclipse.esmf.aspectmodel.resolver.fs.StructuredModelsRoot;
 import org.eclipse.esmf.aspectmodel.urn.AspectModelUrn;

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/FromLoadedFileStrategy.java

@@ -17,6 +17,7 @@
 import java.util.stream.Stream;
 
 import org.eclipse.esmf.aspectmodel.AspectModelFile;
+import org.eclipse.esmf.aspectmodel.resolver.exceptions.ModelResolutionException;
 import org.eclipse.esmf.aspectmodel.urn.AspectModelUrn;
 
 /**

core/esmf-aspect-meta-model-java/src/main/java/org/eclipse/esmf/aspectmodel/resolver/GitHubFileLocation.java

@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2024 Robert Bosch Manufacturing Solutions GmbH
+ *
+ * See the AUTHORS file(s) distributed with this work for additional
+ * information regarding authorship.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+package org.eclipse.esmf.aspectmodel.resolver;
+
+import java.io.Serial;
+import java.util.Arrays;
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import io.vavr.control.Try;
+
+public record GitHubFileLocation(
+      GithubRepository repositoryLocation,
+      String directory,
+      String namespaceMainPart,
+      String version,
+      String filename
+) {
+   public static Optional<GitHubFileLocation> parse( final String url ) {
+      return Try.of( () -> new GitHubUrlParser( url ).get() ).toJavaOptional();
+   }
+
+   private static class ParsingException extends RuntimeException {
+      @Serial
+      private static final long serialVersionUID = -4855068216382713797L;
+   }
+
+   private static class GitHubUrlParser implements Supplier<GitHubFileLocation> {
+      private int index = 0;
+      private final String source;
+
+      private GitHubUrlParser( final String source ) {
+         this.source = source;
+      }
+
+      private String readSection() {
+         final int oldIndex = index;
+         while ( index < source.length() && source.charAt( index ) != '/' ) {
+            index++;
+         }
+         if ( source.charAt( index ) != '/' ) {
+            throw new ParsingException();
+         }
+         final String result = source.substring( oldIndex, index );
+         index++; // eat the slash
+         return result;
+      }
+
+      private void eatToken( final String token ) {
+         if ( source.startsWith( token, index ) ) {
+            index += token.length();
+            return;
+         }
+         throw new ParsingException();
+      }
+
+      @Override
+      public GitHubFileLocation get() {
+         eatToken( "https://" );
+         final String host = readSection();
+         final String owner = readSection();
+         final String repository = readSection();
+         final GithubRepository.Ref ref;
+         if ( source.substring( index ).startsWith( "blob" ) ) {
+            eatToken( "blob/" );
+            final String blob = readSection();
+            ref = blob.matches( "[vV]?\\d+\\.\\d+.*" )
+                  ? new GithubRepository.Tag( blob )
+                  : new GithubRepository.Branch( blob );
+         } else {
+            eatToken( "raw/refs/" );
+            if ( source.substring( index ).startsWith( "heads" ) ) {
+               eatToken( "heads/" );
+               final String branchName = readSection();
+               ref = new GithubRepository.Branch( branchName );
+            } else {
+               eatToken( "tags/" );
+               final String tag = readSection();
+               ref = new GithubRepository.Tag( tag );
+            }
+         }
+         final String rest = source.substring( index );
+         final String[] parts = rest.split( "/" );
+         final String fileName = parts[parts.length - 1];
+         final String version = parts[parts.length - 2];
+         final String namespaceMainPart = parts[parts.length - 3];
+         final String directory = String.join( "/", Arrays.copyOfRange( parts, 0, parts.length - 3 ) );
+         final GithubRepository repoLocation = new GithubRepository( host, owner, repository, ref );
+         return new GitHubFileLocation( repoLocation, directory, namespaceMainPart, version, fileName );
+      }
+   }
+}