Skip to content

Commit

Permalink
Revert multi-modal transpilation output, see #1459
Browse files Browse the repository at this point in the history
  • Loading branch information
@samreid
samreid committed Sep 12, 2024
1 parent 5e62786 commit d477d94
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 153 deletions.
148 changes: 37 additions & 111 deletions js/common/Transpiler.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,6 @@
*
* Additionally, will transpile *.wgsl files to *.js files.
*
* To support the browser and node.js, we output two modes:
* 1. 'js' outputs to chipper/dist/js - import statements, can be launched in the browser
* 2. 'commonjs' outputs to chipper/dist/commonjs - require/module.exports, can be used in node.js
*
* grunt is constrained to use require statements, so that is why we must support the commonjs mode.
*
* See transpile.js for the CLI usage
*
* @author Sam Reid (PhET Interactive Simulations)
Expand Down Expand Up @@ -47,28 +41,6 @@ const subdirs = [ 'js', 'images', 'mipmaps', 'sounds', 'shaders', 'common', 'wgs

const getActiveRepos = () => fs.readFileSync( '../perennial-alias/data/active-repos', 'utf8' ).trim().split( '\n' ).map( sim => sim.trim() );

const getModesForRepo = repo => {

// TODO: Duplicated repos in gruntMain.js https://github.com/phetsims/chipper/issues/1437
const dualRepos = [ 'chipper', 'perennial-alias', 'perennial', 'phet-core' ];
if ( dualRepos.includes( repo ) ) {
return [ 'js', 'commonjs' ];
}
else {
return [ 'js' ];
}
};

/**
* Get a cache status key for the file path and mode
* @param filePath
* @param mode 'js' or 'commonjs'
* @returns {string}
*/
const getStatusKey = ( filePath, mode ) => {
return filePath + ( mode === 'js' ? '@js' : '@commonjs' );
};

class Transpiler {

constructor( options ) {
Expand Down Expand Up @@ -128,33 +100,30 @@ class Transpiler {
/**
* Returns the path in chipper/dist that corresponds to a source file.
* @param filename
* @param mode - 'js' or 'commonjs'
* @returns {string}
* @private
*/
static getTargetPath( filename, mode ) {
assert( mode === 'js' || mode === 'commonjs', 'invalid mode: ' + mode );
static getTargetPath( filename ) {
const relativePath = path.relative( root, filename );
const suffix = relativePath.substring( relativePath.lastIndexOf( '.' ) );

// Note: When we upgrade to Node 16, this may no longer be necessary, see https://github.com/phetsims/chipper/issues/1437#issuecomment-1222574593
// TODO: Get rid of mjs?: https://github.com/phetsims/chipper/issues/1437
const isMJS = relativePath.endsWith( '.mjs' );
// Note: When we upgrade to Node 16, this may no longer be necessary, see https://github.com/phetsims/chipper/issues/1272#issuecomment-1222574593
// TODO: Get rid of mjs: https://github.com/phetsims/chipper/issues/1272
const isMJS = relativePath.includes( 'phet-build-script' ) ||
relativePath.endsWith( '.mjs' );

const extension = isMJS ? '.mjs' : '.js';
return Transpiler.join( root, 'chipper', 'dist', mode, ...relativePath.split( path.sep ) ).split( suffix ).join( extension );
return Transpiler.join( root, 'chipper', 'dist', 'js', ...relativePath.split( path.sep ) ).split( suffix ).join( extension );
}

/**
* Transpile the file (using babel for JS/TS), and write it to the corresponding location in chipper/dist
* @param {string} sourceFile
* @param {string} targetPath
* @param {string} text - file text
* @param {string} mode - 'js' or 'commonjs'
* @private
*/
transpileFunction( sourceFile, targetPath, text, mode ) {
assert( mode === 'js' || mode === 'commonjs', 'invalid mode: ' + mode );
transpileFunction( sourceFile, targetPath, text ) {
let js;
if ( sourceFile.endsWith( '.wgsl' ) ) {
const pathToRoot = '../'.repeat( sourceFile.match( /\//g ).length - 1 );
Expand All @@ -171,8 +140,7 @@ class Transpiler {
// in every sim repo. This strategy is also used in transpile.js
presets: [
'../chipper/node_modules/@babel/preset-typescript',
'../chipper/node_modules/@babel/preset-react',
...( mode === 'js' ? [] : [ [ '../chipper/node_modules/@babel/preset-env', { modules: 'commonjs' } ] ] )
'../chipper/node_modules/@babel/preset-react'
],
sourceMaps: 'inline',

Expand Down Expand Up @@ -207,11 +175,10 @@ class Transpiler {
}

// @public. Delete any files in chipper/dist/js that don't have a corresponding file in the source tree
pruneStaleDistFiles( mode ) {
assert( mode === 'js' || mode === 'commonjs', 'invalid mode: ' + mode );
pruneStaleDistFiles() {
const startTime = Date.now();

const start = `../chipper/dist/${mode}/`;
const start = '../chipper/dist/js/';

const visitFile = path => {
path = Transpiler.forwardSlashify( path );
Expand Down Expand Up @@ -253,7 +220,7 @@ class Transpiler {

const endTime = Date.now();
const elapsed = endTime - startTime;
console.log( `Clean stale chipper/dist/${mode} files finished in ` + elapsed + 'ms' );
console.log( 'Clean stale chipper/dist/js files finished in ' + elapsed + 'ms' );
}

// @public join and normalize the paths (forward slashes for ease of search and readability)
Expand All @@ -263,11 +230,9 @@ class Transpiler {

/**
* @param {string} filePath
* @param {string} mode - 'js' or 'commonjs'
* @private
*/
visitFileWithMode( filePath, mode ) {
assert( mode === 'js' || mode === 'commonjs', 'invalid mode: ' + mode );
visitFile( filePath ) {
if ( _.some( [ '.js', '.ts', '.tsx', '.wgsl', '.mjs' ], extension => filePath.endsWith( extension ) ) &&
!this.isPathIgnored( filePath ) ) {

Expand All @@ -278,37 +243,26 @@ class Transpiler {
// If the file has changed, transpile and update the cache. We have to choose on the spectrum between safety
// and performance. In order to maintain high performance with a low error rate, we only write the transpiled file
// if (a) the cache is out of date (b) there is no target file at all or (c) if the target file has been modified.
const targetPath = Transpiler.getTargetPath( filePath, mode );
const targetPath = Transpiler.getTargetPath( filePath );

const statusKey = getStatusKey( filePath, mode );

if (
!this.status[ statusKey ] ||
this.status[ statusKey ].sourceMD5 !== hash ||
!fs.existsSync( targetPath ) ||
this.status[ statusKey ].targetMilliseconds !== Transpiler.modifiedTimeMilliseconds( targetPath )
) {
if ( !this.status[ filePath ] || this.status[ filePath ].sourceMD5 !== hash || !fs.existsSync( targetPath ) || this.status[ filePath ].targetMilliseconds !== Transpiler.modifiedTimeMilliseconds( targetPath ) ) {

try {
let reason = '';
if ( this.verbose ) {
reason = ( !this.status[ statusKey ] ) ? ' (not cached)' :
( this.status[ statusKey ].sourceMD5 !== hash ) ? ' (changed)' :
( !fs.existsSync( targetPath ) ) ? ' (no target)' :
( this.status[ statusKey ].targetMilliseconds !== Transpiler.modifiedTimeMilliseconds( targetPath ) ) ? ' (target modified)' :
'???';
reason = ( !this.status[ filePath ] ) ? ' (not cached)' : ( this.status[ filePath ].sourceMD5 !== hash ) ? ' (changed)' : ( !fs.existsSync( targetPath ) ) ? ' (no target)' : ( this.status[ filePath ].targetMilliseconds !== Transpiler.modifiedTimeMilliseconds( targetPath ) ) ? ' (target modified)' : '???';
}
this.transpileFunction( filePath, targetPath, text, mode );
this.transpileFunction( filePath, targetPath, text );

this.status[ statusKey ] = {
this.status[ filePath ] = {
sourceMD5: hash,
targetMilliseconds: Transpiler.modifiedTimeMilliseconds( targetPath )
};
fs.writeFileSync( statusPath, JSON.stringify( this.status, null, 2 ) );
const now = Date.now();
const nowTimeString = new Date( now ).toLocaleTimeString();

!this.silent && console.log( `${nowTimeString}, ${( now - changeDetectedTime )} ms: ${filePath} ${mode}${reason}` );
!this.silent && console.log( `${nowTimeString}, ${( now - changeDetectedTime )} ms: ${filePath}${reason}` );
}
catch( e ) {
console.log( e );
Expand All @@ -318,21 +272,8 @@ class Transpiler {
}
}

/**
* For *.ts and *.js files, checks if they have changed file contents since last transpile. If so, the
* file is transpiled.
* @param {string} filePath
* @param {string[]} modes - some of 'js','commonjs'
* @private
*/
visitFile( filePath, modes ) {
assert( Array.isArray( modes ), 'invalid modes: ' + modes );
modes.forEach( mode => this.visitFileWithMode( filePath, mode ) );
}

// @private - Recursively visit a directory for files to transpile
visitDirectory( dir, modes ) {
assert( Array.isArray( modes ), 'invalid modes: ' + modes );
visitDirectory( dir ) {
if ( fs.existsSync( dir ) ) {
const files = fs.readdirSync( dir );
files.forEach( file => {
Expand All @@ -341,10 +282,10 @@ class Transpiler {
assert( !child.endsWith( '/dist' ), 'Invalid path: ' + child + ' should not be in dist directory.' );

if ( fs.lstatSync( child ).isDirectory() ) {
this.visitDirectory( child, modes );
this.visitDirectory( child );
}
else {
this.visitFile( child, modes );
this.visitFile( child );
}
} );
}
Expand Down Expand Up @@ -395,35 +336,29 @@ class Transpiler {
repos.forEach( repo => this.transpileRepo( repo ) );
}

// @public - Visit all the subdirectories in a repo that need transpilation for the specified modes
transpileRepoWithModes( repo, modes ) {
assert( Array.isArray( modes ), 'modes should be an array' );
subdirs.forEach( subdir => this.visitDirectory( Transpiler.join( '..', repo, subdir ), modes ) );
// @public - Visit all the subdirectories in a repo that need transpilation
transpileRepo( repo ) {
subdirs.forEach( subdir => this.visitDirectory( Transpiler.join( '..', repo, subdir ) ) );
if ( repo === 'sherpa' ) {

// Our sims load this as a module rather than a preload, so we must transpile it
this.visitFile( Transpiler.join( '..', repo, 'lib', 'game-up-camera-1.0.0.js' ), modes );
this.visitFile( Transpiler.join( '..', repo, 'lib', 'pako-2.0.3.min.js' ), modes ); // used for phet-io-wrappers tests
this.visitFile( Transpiler.join( '..', repo, 'lib', 'big-6.2.1.mjs' ), modes ); // for consistent, cross-browser number operations (thanks javascript)
this.visitFile( Transpiler.join( '..', repo, 'lib', 'game-up-camera-1.0.0.js' ) );
this.visitFile( Transpiler.join( '..', repo, 'lib', 'pako-2.0.3.min.js' ) ); // used for phet-io-wrappers tests
this.visitFile( Transpiler.join( '..', repo, 'lib', 'big-6.2.1.mjs' ) ); // for consistent, cross-browser number operations (thanks javascript)
Object.keys( webpackGlobalLibraries ).forEach( key => {
const libraryFilePath = webpackGlobalLibraries[ key ];
this.visitFile( Transpiler.join( '..', ...libraryFilePath.split( '/' ) ), modes );
this.visitFile( Transpiler.join( '..', ...libraryFilePath.split( '/' ) ) );
} );
}
else if ( repo === 'brand' ) {
this.visitDirectory( Transpiler.join( '..', repo, 'phet' ), modes );
this.visitDirectory( Transpiler.join( '..', repo, 'phet-io' ), modes );
this.visitDirectory( Transpiler.join( '..', repo, 'adapted-from-phet' ), modes );
this.visitDirectory( Transpiler.join( '..', repo, 'phet' ) );
this.visitDirectory( Transpiler.join( '..', repo, 'phet-io' ) );
this.visitDirectory( Transpiler.join( '..', repo, 'adapted-from-phet' ) );

this.brands.forEach( brand => this.visitDirectory( Transpiler.join( '..', repo, brand ), modes ) );
this.brands.forEach( brand => this.visitDirectory( Transpiler.join( '..', repo, brand ) ) );
}
}

// @public - Visit all the subdirectories in a repo that need transpilation
transpileRepo( repo ) {
this.transpileRepoWithModes( repo, getModesForRepo( repo ) );
}

// @public
transpileAll() {
this.transpileRepos( [ ...this.activeRepos, ...this.repos ] );
Expand Down Expand Up @@ -486,25 +421,17 @@ class Transpiler {
const pathExists = fs.existsSync( filePath );

if ( !pathExists ) {

// TODO: This should be a constant for "all supported modes" https://github.com/phetsims/chipper/issues/1437
const modes = [ 'js', 'commonjs' ];

modes.forEach( mode => {
const targetPath = Transpiler.getTargetPath( filePath, mode );
const targetPath = Transpiler.getTargetPath( filePath );
if ( fs.existsSync( targetPath ) && fs.lstatSync( targetPath ).isFile() ) {
fs.unlinkSync( targetPath );

const statusKey = getStatusKey( filePath, mode );

delete this.status[ statusKey ];
delete this.status[ filePath ];
this.saveCache();
const now = Date.now();
const reason = ' (deleted)';

!this.silent && console.log( `${new Date( now ).toLocaleTimeString()}, ${( now - changeDetectedTime )} ms: ${filePath}${mode}${reason}` );
!this.silent && console.log( `${new Date( now ).toLocaleTimeString()}, ${( now - changeDetectedTime )} ms: ${filePath}${reason}` );
}
} );

return;
}
Expand All @@ -523,10 +450,9 @@ class Transpiler {
}
else {
const terms = filename.split( path.sep );
const repo = terms[ 0 ];
if ( ( this.activeRepos.includes( repo ) || this.repos.includes( repo ) )
if ( ( this.activeRepos.includes( terms[ 0 ] ) || this.repos.includes( terms[ 0 ] ) )
&& subdirs.includes( terms[ 1 ] ) && pathExists ) {
this.visitFile( filePath, getModesForRepo( repo ) );
this.visitFile( filePath );
}
}
} );
Expand Down
45 changes: 3 additions & 42 deletions js/grunt/gruntMain.js
View file
Original file line number Diff line number Diff line change
@@ -1,51 +1,12 @@
// Copyright 2024, University of Colorado Boulder

/**
* gruntMain.js is the entry point for launching the grunt tasks for the PhET build system.
*
* In order to support development with TypeScript for the grunt tasks, transpile before starting and then point
* to the transpiled version.
*
* Since this is the entry point for TypeScript, it and its dependencies must remain in JavaScript.
* gruntMain.js is the entry point for launching the grunt tasks for the PhET build system. Individual sims and repos
* point here for flexibility in the build system entry.
*
* @author Sam Reid (PhET Interactive Simulations)
*/

/* eslint-env node */

// Switch between TypeScript and JavaScript (feature flag)
const LAUNCH_FROM_CHIPPER_DIST = false;

if ( LAUNCH_FROM_CHIPPER_DIST ) {

const Transpiler = require( '../../js/common/Transpiler' );

const commonJSTranspiler = new Transpiler();

/**
Transpile the entry points
These files may have already been transpiled by a watch process. In that case, these lines are a no-op.
For build servers and other processes that do not have a watch process, or if a developer watch process is not running
or out-of-date, this will transpile synchronously during startup.
If we forgot to transpile something, we will get a module not found runtime error, and
can add more entry points to this list.
Note that 2 Transpile processes trying to write the same file at the same time may corrupt the file, since
we do not have atomic writes.
*/
// TODO: should this be silent? https://github.com/phetsims/chipper/issues/1437
commonJSTranspiler.transpileRepoWithModes( 'chipper', [ 'commonjs' ] );
commonJSTranspiler.transpileRepoWithModes( 'phet-core', [ 'commonjs' ] );
commonJSTranspiler.transpileRepoWithModes( 'perennial-alias', [ 'commonjs' ] );
commonJSTranspiler.transpileRepoWithModes( 'perennial', [ 'commonjs' ] ); // TODO: needed by aqua/ https://github.com/phetsims/chipper/issues/1437
commonJSTranspiler.saveCache();

// TODO: Make sure the above repos are covered by tsconfig/all, see https://github.com/phetsims/chipper/issues/1437

// use chipper's gruntfile
module.exports = require( '../../dist/commonjs/chipper/js/grunt/Gruntfile.js' );
}
else {
module.exports = require( './Gruntfile.js' );
}
module.exports = require( './Gruntfile.js' );

0 comments on commit d477d94

Please sign in to comment.