Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: OrphanedShapes TestModel #744

Open
wants to merge 39 commits into
base: main-1.x
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
92440a6
m
lucasmcdonald3 Nov 22, 2024
60160ab
m
lucasmcdonald3 Nov 22, 2024
2f49bfc
m
lucasmcdonald3 Nov 22, 2024
080a0d7
m
lucasmcdonald3 Nov 22, 2024
ea69572
m
lucasmcdonald3 Nov 22, 2024
e01c79d
m
lucasmcdonald3 Nov 22, 2024
56a1be6
m
lucasmcdonald3 Nov 22, 2024
39e8c5e
POCs for net, java, python
lucasmcdonald3 Dec 5, 2024
a73fee0
m
lucasmcdonald3 Nov 22, 2024
8df6dd4
merge
lucasmcdonald3 Dec 5, 2024
a30f9d9
java python pass
lucasmcdonald3 Dec 5, 2024
b506b65
rust
lucasmcdonald3 Dec 9, 2024
96c568f
m
lucasmcdonald3 Nov 22, 2024
398e899
merge
lucasmcdonald3 Dec 9, 2024
5ff1750
go
lucasmcdonald3 Dec 10, 2024
fd8164c
m
lucasmcdonald3 Dec 11, 2024
20a00f7
m
lucasmcdonald3 Dec 11, 2024
6816df5
m
lucasmcdonald3 Dec 11, 2024
8aae719
m
lucasmcdonald3 Dec 11, 2024
b03fc10
m
lucasmcdonald3 Dec 11, 2024
4df19f1
m
lucasmcdonald3 Dec 11, 2024
d5cafd5
nogo
lucasmcdonald3 Dec 11, 2024
daae9d9
m
lucasmcdonald3 Dec 11, 2024
20cc166
m
lucasmcdonald3 Dec 11, 2024
c77ffab
Merge branch 'main-1.x' into orphaned-shapes
lucasmcdonald3 Dec 11, 2024
a1af584
m
lucasmcdonald3 Dec 11, 2024
f8eaf5e
m
lucasmcdonald3 Dec 11, 2024
b4bb57a
format
lucasmcdonald3 Dec 11, 2024
4e39e8c
(c)
lucasmcdonald3 Dec 11, 2024
a049cd7
311
lucasmcdonald3 Dec 11, 2024
6cf63f0
Merge branch 'main-1.x' into orphaned-shapes
lucasmcdonald3 Dec 11, 2024
1f91128
m
lucasmcdonald3 Dec 11, 2024
54ef386
m
lucasmcdonald3 Dec 12, 2024
f2488ce
m
lucasmcdonald3 Dec 12, 2024
5dc6296
m
lucasmcdonald3 Dec 12, 2024
2366eab
Merge branch 'main-1.x' into orphaned-shapes
lucasmcdonald3 Dec 19, 2024
b92e7b7
Merge branch 'main-1.x' into orphaned-shapes
lucasmcdonald3 Jan 6, 2025
25a49fc
Merge branch 'main-1.x' into orphaned-shapes
lucasmcdonald3 Jan 7, 2025
c93de72
validation
lucasmcdonald3 Jan 7, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions TestModels/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,23 @@
# Polymorph Generated Python
**/runtimes/python/src/**/smithygenerated/

# Polymorph Generated Rust
lucasmcdonald3 marked this conversation as resolved.
Show resolved Hide resolved
**/src/client.rs
**/src/client
**/src/conversions.rs
**/src/conversions
**/src/deps.rs
**/src/deps
**/src/error.rs
**/src/error
**/src/operation.rs
**/src/operation
**/src/standard_library_conversions.rs
**/src/standard_library_externs.rs
**/src/types.rs
**/src/types
**/target

# .NET Artifacts
**/bin
**/obj
Expand Down
55 changes: 55 additions & 0 deletions TestModels/OrphanedShapes/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0

CORES=2

ENABLE_EXTERN_PROCESSING=1
TRANSPILE_TESTS_IN_RUST=1

include ../SharedMakefile.mk

NAMESPACE=simple.orphaned

PROJECT_SERVICES := \
SimpleOrphaned

MAIN_SERVICE_FOR_RUST := SimpleOrphaned

SERVICE_NAMESPACE_SimpleOrphaned=simple.orphaned

SERVICE_DEPS_SimpleOrphaned :=

SMITHY_DEPS=dafny-dependencies/Model/traits.smithy

# This project has no dependencies
# DEPENDENT-MODELS:=

POLYMORPH_OPTIONS=--generate project-files,client-constructors\

# Go

GO_MODULE_NAME="github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes"

TRANSLATION_RECORD_GO := \
dafny-dependencies/StandardLibrary/runtimes/go/ImplementationFromDafny-go/ImplementationFromDafny-go.dtr

# Python

PYTHON_MODULE_NAME=simple_orphaned

TRANSLATION_RECORD_PYTHON := \
--translation-record ../dafny-dependencies/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr

# Constants for languages that drop extern names (Python, Go)

TYPES_FILE_PATH=Model/SimpleOrphanedTypes.dfy
TYPES_FILE_WITH_EXTERN_STRING="module {:extern \"simple.orphaned.internaldafny.types\" } SimpleOrphanedTypes"
TYPES_FILE_WITHOUT_EXTERN_STRING="module SimpleOrphanedTypes"

INDEX_FILE_PATH=src/Index.dfy
INDEX_FILE_WITH_EXTERN_STRING="module {:extern \"simple.orphaned.internaldafny\" } SimpleOrphaned refines AbstractSimpleOrphanedService {"
INDEX_FILE_WITHOUT_EXTERN_STRING="module SimpleOrphaned refines AbstractSimpleOrphanedService {"

WRAPPED_INDEX_FILE_PATH=src/WrappedSimpleOrphanedImpl.dfy
WRAPPED_INDEX_FILE_WITH_EXTERN_STRING="module {:options \"--function-syntax:4\"} {:extern \"simple.orphaned.internaldafny.wrapped\"} WrappedSimpleOrphanedService refines WrappedAbstractSimpleOrphanedService {"
WRAPPED_INDEX_FILE_WITHOUT_EXTERN_STRING="module {:options \"--function-syntax:4\"} WrappedSimpleOrphanedService refines WrappedAbstractSimpleOrphanedService {"
130 changes: 130 additions & 0 deletions TestModels/OrphanedShapes/Model/OrphanedShapes.smithy
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

$version: "2"
namespace simple.orphaned

@aws.polymorph#localService(
sdkId: "SimpleOrphaned",
config: SimpleOrphanedConfig,
)
service SimpleOrphaned {
version: "2021-11-01",
resources: [],
operations: [],
}

structure SimpleOrphanedConfig {
structureMember: OrphanedConfigShape
}

structure OrphanedConfigShape {
stringMember: String
}

blob OrphanedBlob

boolean OrphanedBoolean

string OrphanedString

// TODO: Once SimpleTypes for commented-out shapes are completed,
// uncomment these and add as members to OrphanedStructure
// byte OrphanedByte

// short OrphanedShort

integer OrphanedInteger

long OrphanedLong

// float OrphanedFloat

// double OrphanedDouble

// bigInteger OrphanedBigInteger

// bigDecimal OrphanedBigDecimal

// timestamp OrphanedTimestamp

// document OrphanedDocument

// This is a smithy V1 Enum
@enum([
{
name: "FIRST",
value: "0x0014",
},
{
name: "SECOND",
value: "0x0046",
},
{
name: "THIRD",
value: "0x0078",
},
])
string OrphanedV1Enum

list OrphanedList {
member: String
}

map OrphanedMap {
key: String
value: String
}

structure OrphanedStructure {
blobValue: OrphanedBlob,
booleanValue: OrphanedBoolean,
stringValue: OrphanedString,
// byteValue: OrphanedByte,
// shortValue: OrphanedShort,
integerValue: OrphanedInteger,
longValue: OrphanedLong,
// floatValue: OrphanedFloat,
// doubleValue: OrphanedDouble,
// bigIntegerValue: OrphanedBigInteger,
// bigDecimalValue: OrphanedBigDecimal,
// timestampValue: OrphanedTimestamp,
unionValue: OrphanedUnion,
enumValue: OrphanedV1Enum,
mapValue: OrphanedMap,
listValue: OrphanedList
}

union OrphanedUnion {
integerValue: Integer
stringValue: String
}

@error("client")
structure OrphanedError {
@required
message: String,
}

@aws.polymorph#reference(resource: OrphanedResource)
structure OrphanedResourceReference {}

@aws.polymorph#extendable
resource OrphanedResource {
operations: [
OrphanedResourceOperation
]
}

operation OrphanedResourceOperation {
input: OrphanedResourceOperationInput
output: OrphanedResourceOperationOutput
}

structure OrphanedResourceOperationInput {
someString: String
}

structure OrphanedResourceOperationOutput {
someString: String
}
57 changes: 57 additions & 0 deletions TestModels/OrphanedShapes/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# OrphanedShapes

## Background
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Most of this content should exist outside of a specific TestModel IMO. I'd suggest moving a good chunk into a separate file under designs.

I'd also suggest naming the complete list of use cases that leads us to deviate from Smithy-Core's approach, just to be clear about which are fundamental (like config shapes) and which are closer to bugs in our implementations.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can do some content moving.

I don't like writing a complete list anywhere. It's a long list (I'd estimate 30+) and it will get stale quickly. This list has recently gotten longer due to Tony's mutations changes.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

30+ individual reasons we need orphaned shapes? Yikes!

I guess what you're saying is: if I wanted to get to a place where local services didn't need generation for orphaned shapes, I'd be better off building that and testing it against all the existing libraries to identify gaps?

(Thanks for satisfying my curiousity on these question btw, I have no intention of blocking this PR as we clearly should be explicitly testing a feature we depend on, I just want to take the opportunity to understand the scope of the feature better, since you would have had to dive into it a certain depth anyway)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think so.

I think we're landing on the idea of the "ideal state", which would be using Smithy-Core's shape discovery logic plus a bounded, limited set of orphaned shapes (like local service config shapes).
And the path to get there may just be a breaking change to Smithy-Dafny (made by testing against complex libraries) combined with Smithy model changes to downstream libraries


This project tests for support of "orphaned" shapes.

An "orphaned" is a shape that exists in a Smithy model, but is not discovered by Smithy-Core's shape discovery logic.

Smithy-Core will only discover and generate code for shapes that are attached to a service via the service's

1. Operations
2. Mixins
3. Errors

and recursive traversals of the connected shapes.

If a shape is defined in a Smithy model,
but not declared in a LocalService's operations or errors
(or recursive traversals of these),
codegens that use Smithy-Core's shape discovery will not generate code for the shape.
(Similar for mixins, but Polymorph doesn't have any usage of these at the moment.)

Smithy-Dafny .NET and Java generate code for orphaned shapes because these codegens don't use Smithy-Core's shape discovery logic.

By default, any language extending a "real" Smithy code generator use Smithy-Core's shape discovery logic, and do not support orphaned shapes.

The expectation is that code is generated for orphaned shapes.
The generated code must include any class definitions and Dafny/native conversions.

## Prerequisites

This TestModel assumes these other TestModels are passing:

- Extern
- Extendable
- Errors
- LocalService
- Union
- Aggregate
- Enum

and these TestModels' prerequisite TestModels.

## Coverage

This TestModel tests some instances of orphaned shapes

- LocalService Config shapes. (Config shapes are "orphaned", but are likely already handled as one-offs by any codegen that's this TestModels' prerequisites)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the main "legit" reason to deviate at least, and I'm hoping in the future we can model local service client creation as an operational as well somehow, since then it wouldn't be a special case.

- Errors
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are these orphaned? Is it because errors from dependency services are used but not directly connected?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Errors will be orphaned if they are not attached to any operations

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, but I assume we still need conversions for them for some reason, just trying to understand that better

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In my mind, it's because Java generates conversions for orphaned errors, so other languages should also generate conversions for parity.

- Resources (with @aws.polymorph#reference trait) and their operations
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These should still be listed as resources on the local service though, right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe, but in many cases they aren't

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

(I also believe that Smithy-Core doesn't discover resources attached to the service -- it's only mixins, operations, and errors)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That wouldn't surprise me given no other code generators produce anything for resources yet. I would still argue Smithy-Core probably should traverse them even if no one else does anything for them yet, so I might try to submit that change to Smithy-Core down the road...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

- Structures (and structures' members)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Which structures other than config structures?

Copy link
Contributor Author

@lucasmcdonald3 lucasmcdonald3 Dec 19, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only examples are unused structures in the MPL.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Which ones specifically? I don't yet appreciate the use case

Copy link
Contributor Author


The bar this TestModel establishes is "if a shape is defined in a Smithy model, its native/Dafny shapes and their conversions are defined in a runtime."

This TestModel asserts this by requiring externs to pass orphaned shapes across the Dafny layer.
The extern implementations MUST call Polymorph-generated native shapes and Dafny/native conversions for these orphaned shapes.
Polymorph will only generate these shapes and conversions if the language's codegen supports orphaned shapes.
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
module github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes

go 1.23.0

require (
github.com/dafny-lang/DafnyRuntimeGo/v4 v4.9.1
github.com/dafny-lang/DafnyStandardLibGo v0.0.0
)

replace github.com/dafny-lang/DafnyStandardLibGo => ../../../../dafny-dependencies/StandardLibrary/runtimes/go/ImplementationFromDafny-go/
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package ExternDefinitions

import (
"fmt"

Wrappers "github.com/dafny-lang/DafnyStandardLibGo/Wrappers"
OrphanedResource "github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes/OrphanedResource"
SimpleOrphanedTypes "github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes/SimpleOrphanedTypes"
simpleorphanedsmithygenerated "github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes/simpleorphanedsmithygenerated"
simpleorphanedsmithygeneratedtypes "github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes/simpleorphanedsmithygeneratedtypes"
)

var _ Wrappers.Dummy__

// TODO: Finish implementation.
// This is missing structure converter.

func (CompanionStruct_Default___) InitializeOrphanedStructure(input SimpleOrphanedTypes.OrphanedStructure) SimpleOrphanedTypes.OrphanedStructure {
// Missing Structure converter
return input
}

func (CompanionStruct_Default___) CallNativeOrphanedResource(input *OrphanedResource.OrphanedResource) Wrappers.Result {
native_resource := simpleorphanedsmithygenerated.OrphanedResource_FromDafny(input)
someString := "the extern MUST provide this string to the native resource's operation"
native_output, err := native_resource.OrphanedResourceOperation(simpleorphanedsmithygeneratedtypes.OrphanedResourceOperationInput{SomeString: &someString})
if err != nil {
return Wrappers.Companion_Result_.Create_Failure_(err)
}
dafny_output := simpleorphanedsmithygenerated.OrphanedResourceOperationOutput_ToDafny(*native_output)
return Wrappers.Companion_Result_.Create_Success_(dafny_output)
}

func (CompanionStruct_Default___) CallNativeOrphanedError(input SimpleOrphanedTypes.Error) SimpleOrphanedTypes.Error {
native_error := simpleorphanedsmithygenerated.Error_FromDafny(input)
native_error.Message = "the extern MUST set this string using the catch-all error converter, NOT the orphaned error-specific converter"
dafny_error_again := simpleorphanedsmithygenerated.Error_ToDafny(native_error)
return dafny_error_again
}
14 changes: 14 additions & 0 deletions TestModels/OrphanedShapes/runtimes/go/TestsFromDafny-go/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes/test

go 1.23.0

require github.com/dafny-lang/DafnyStandardLibGo v0.0.0

require (
github.com/dafny-lang/DafnyRuntimeGo/v4 v4.9.1
github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes v0.0.0
)

replace github.com/smithy-lang/smithy-dafny/TestModels/OrphanedShapes v0.0.0 => ../ImplementationFromDafny-go

replace github.com/dafny-lang/DafnyStandardLibGo => ../../../../dafny-dependencies/StandardLibrary/runtimes/go/ImplementationFromDafny-go/
Loading
Loading