Merge pull request #1 from TelosLabs/feature/development_mode_streaming

Feature/development mode streaming

.gitignore

@@ -9,4 +9,6 @@
 
 # rspec failure tracking
 .rspec_status
+/.env.*
 
+.DS_Store

Gemfile

@@ -5,6 +5,7 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in openlayer.gemspec
 gemspec
 
+gem "pry", "~> 0.14"
 gem "rake", "~> 13.0"
 
 gem "rspec", "~> 3.0"

Gemfile.lock

@@ -1,26 +1,39 @@
 PATH
   remote: .
   specs:
-    openlayer (0.2.0)
+    openlayer (0.2.2)
+      date
       faraday (~> 2.9.0)
+      faraday-multipart
+      yaml
+      zlib
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
+    coderay (1.1.3)
+    date (3.3.4)
     diff-lcs (1.5.0)
     faraday (2.9.0)
       faraday-net_http (>= 2.0, < 3.2)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
     faraday-net_http (3.1.0)
       net-http
     json (2.7.1)
     language_server-protocol (3.17.0.3)
+    method_source (1.0.0)
+    multipart-post (2.4.0)
     net-http (0.4.1)
       uri
     parallel (1.24.0)
     parser (3.3.0.4)
       ast (~> 2.4.1)
       racc
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
     racc (1.7.3)
     rainbow (3.1.1)
     rake (13.1.0)
@@ -55,12 +68,18 @@ GEM
     ruby-progressbar (1.13.0)
     unicode-display_width (2.5.0)
     uri (0.13.0)
+    yaml (0.3.0)
+    zlib (3.1.0)
 
 PLATFORMS
   arm64-darwin-21
+  arm64-darwin-22
+  arm64-darwin-23
+  x86_64-linux
 
 DEPENDENCIES
   openlayer!
+  pry (~> 0.14)
   rake (~> 13.0)
   rspec (~> 3.0)
   rubocop (~> 1.21)

README.md

@@ -13,6 +13,7 @@ If bundler is not being used to manage dependencies, install the gem by executin
 
 ## Usage
 
+### Inference Monitoring Mode
 ```ruby
 client = Openlayer::Client.new(api_key: "YOUR_OPENLAYER_API_KEY")
 inference_pipeline = client.inference_pipeline("YOUR_INFERENCE_PIPELINE_ID")
@@ -24,6 +25,39 @@ inference_pipeline.stream_data(
 )
 ```
 
+
+
+### Projects
+#### Load Project
+Loads an existing project from the Openlayer platform.
+
+```ruby
+client = Openlayer::Client.new(api_key: "YOUR_OPENLAYER_API_KEY")
+project = client.load_project(name: "your_project_name")
+```
+
+#### Development Mode for CI/CD
+```ruby
+client = Openlayer::Client.new(api_key: "YOUR_OPENLAYER_API_KEY")
+project = client.load_project(name: "your_project_name")
+
+
+project.add_dataset(file_path: "DATASET_FILE_PATH", dataset_config_file_path: "DATASET_CONFIG_FILE_PATH")
+project.add_model(model_config_file_path: "MODEL_CONFIG_PATH")
+project.commit(message: "This is a commit message")
+version = project.push
+
+version.wait_for_completion(timeout: 500)
+version.print_test_report
+
+if version.failing_test_count > 0:
+    puts "Failing pipeline due to failing goals."
+    exit(1)
+```
+
+- Check the example in `examples/staging` for what the YAMLs look like.
+- Check the [Dataset Config](https://docs.openlayer.com/how-to-guides/write-dataset-configs/llm-dataset-config) and the [Model Config](https://docs.openlayer.com/how-to-guides/write-model-configs/llm-config) docs on Openlayer to see additional attributes for the configs.
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

bin/console

@@ -3,11 +3,9 @@
 
 require "bundler/setup"
 require "openlayer"
-
+require "pry"
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
 
-Openlayer::Client.new(api_key: "api_key")
-
-require "irb"
-IRB.start(__FILE__)
+client = Openlayer::Client.new(api_key: ENV.fetch("OPENLAYER_API_KEY"))
+Pry.start(binding)

examples/staging/commit.yaml

@@ -0,0 +1,2 @@
+date: Sat Feb 25 14:24:18 2023
+message: Initial commit!

examples/staging/model/model_config.yaml

@@ -0,0 +1,12 @@
+architectureType: llm
+inputVariableNames:
+  - input
+metadata: {}
+model: gpt-3.5-turbo
+modelParameters:
+  temperature: 0.7
+modelProvider: OpenAI
+modelType: api
+name: AI assistant
+prompt: 
+  - {"role": "user", "content": "{{ input }}"}

examples/staging/validation/dataset_config.yaml

@@ -0,0 +1,12 @@
+columnNames:
+  - input
+  - output
+  - ground_truth
+groundTruthColumnName: ground_truth
+inputVariableNames:
+  - input
+label: validation
+language: en
+metadata: {}
+outputColumnName: output
+sep: ","

lib/openlayer.rb

@@ -5,7 +5,12 @@
 module Openlayer
   autoload :Client, "openlayer/client"
   autoload :Error, "openlayer/error"
+  autoload :Object, "openlayer/object"
   autoload :InferencePipeline, "openlayer/inference_pipeline"
+  autoload :S3PresignedClient, "openlayer/services/s3_presigned_client"
+  autoload :TarFileHelper, "openlayer/services/tar_file_helper"
 
-  # Your code goes here...
+  # Classes used to return a nicer object wrapping the response data
+  autoload :Project, "openlayer/objects/project"
+  autoload :ProjectVersion, "openlayer/objects/project_version"
 end

lib/openlayer/client.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "faraday"
+require "faraday/multipart"
 
 module Openlayer
   class Client
@@ -14,6 +15,25 @@ def initialize(api_key:, adapter: Faraday.default_adapter, stubs: nil)
       @stubs = stubs
     end
 
+    def inference_pipeline(inference_pipeline_id)
+      InferencePipeline.new(self, inference_pipeline_id)
+    end
+
+    def load_project(name:)
+      Project.from_response self, connection.get("projects", project_search_params(name: name))
+    end
+
+    def presigned_url(object_name: nil, workspace_id: nil)
+      handle_response connection.post(
+        "storage/presigned-url?objectName=staging", presigned_url_params(object_name: object_name,
+                                                                         workspace_id: workspace_id)
+      )
+    end
+
+    def load_project_version(id:, payload: {})
+      ProjectVersion.from_response self, connection.post("projects/#{id}/versions", payload)
+    end
+
     def connection
       @connection ||= Faraday.new do |conn|
         conn.url_prefix = BASE_URL
@@ -24,8 +44,34 @@ def connection
       end
     end
 
-    def inference_pipeline(inference_pipeline_id)
-      InferencePipeline.new(self, inference_pipeline_id)
+    private
+
+    def project_search_params(name:)
+      { version: "0.1.0a25", name: name }
+    end
+
+    def presigned_url_params(object_name: nil, workspace_id: nil)
+      { "objectName": object_name, "workspaceId": workspace_id, version: "0.1.0a25" }.compact
+    end
+
+    def handle_response(response)
+      message = response.body["error"]
+      case response.status
+      when 200
+        response.body
+      when 401
+        raise Error, message
+      when 404
+        raise Error, message
+      when 422
+        raise Error, message
+      when 500
+        raise Error, message
+      when 200..299
+        response.body
+      else
+        raise Error, message
+      end
     end
   end
 end

lib/openlayer/object.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "ostruct"
+module Openlayer
+  class Object
+    def initialize(attributes)
+      @attributes = OpenStruct.new(attributes)
+    end
+
+    def method_missing(method, *args, &block)
+      method = snake_to_camel(method.to_s)
+      attribute = @attributes.send(method, *args, &block)
+      attribute.is_a?(Hash) ? Object.new(attribute) : attribute
+    end
+
+    def respond_to_missing?(_method, _include_private = false)
+      true
+    end
+
+    private
+
+    def snake_to_camel(snake_str)
+      snake_str.split("_").each_with_index.map do |part, index|
+        index.zero? ? part : part.capitalize
+      end.join
+    end
+  end
+end