Implement screen capture, save, open, and search.

[octalmage]

Just an issue to track progress and ideas.

[octalmage]

For the bitmap functions, I'll return an object similar to this:

MMBitmapRef createMMBitmap(uint8_t *buffer,
                           size_t width,
                           size_t height,
                           size_t bytewidth,
                           uint8_t bitsPerPixel,
                           uint8_t bytesPerPixel)
{
    MMBitmapRef bitmap = malloc(sizeof(MMBitmap));
    if (bitmap == NULL) return NULL;

    bitmap->imageBuffer = buffer;
    bitmap->width = width;
    bitmap->height = height;
    bitmap->bytewidth = bytewidth;
    bitmap->bitsPerPixel = bitsPerPixel;
    bitmap->bytesPerPixel = bytesPerPixel;

    return bitmap;
}

This way we can pass the necessary

[octalmage]

With the new syntax this will go in the index.js.

function bitmap (imagedata, width, height) 
{
    this.imagedata = imagedata;
    this.width = width;
    this.height. = height;
}

bitmap.prototype.save = function(path) 
{
    robotjs.saveBitmap(this.imagedata, path);
};

module.exports.screen.capture = function()
{
    b = robotjs.captureScreen();
    return new bitmap(b.imagedata, b.width, b.height);
}

Then you'll use it like this:

var robot = require("robotjs");

var desktop = robot.screen.capture();

desktop.save("screencapture.png");

Note: I wrote this on my phone.

[octalmage]

Might need to use:

createStringFromMMBitmap
createMMBitmapFromString

To pass the image between V8 and JavaScript.

[Deltatiger]

This should be the next priority. Capture screen will allow for more projects to adopt RobotJS. (Also my mini project of a remote desktop )

[octalmage]

Agreed, I would like to implement this since there's a special way I want it implemented (see bitmap branch). I'll start on this after #87 is merged and RobotJS is completely updated. This is going to take some extra work on Windows due to the libpng and zlib dependencies, which is the main reason I put it off. These libraries are required for disk related functionality, and createMMBitmapFromString.

#64 is another priority for me.

[octalmage]

Here's the latest:

http://blog.robotjs.io/post/130642552013/i-spent-a-few-hours-yesterday-and-today-working-on

[Deltatiger]

@octalmage I will look into this bug and get it sorted out. Is the version implemented for Linux ?

[octalmage]

Nice! Yeah it works for Linux too. I'm pretty sure my math is wrong for the buffers though. If I remember correctly it crashed when I passed the image buffer back to C++ from JavaScript in colorAt.

[SiriusDG]

Is the image save implemented yet? I tried to follow the demo from comments, desktop.save did not yet work. Any simple way to save captured images? This is about all I really need to make great use of this, thanks!

[octalmage]

The first part is implemented, screen capturing, but not saving yet. We just released the capture code in 0.4.0 this weekend, so it will be a bit longer before the saving code is out.

[roccomuso]

+1

[stephenthedev]

@octalmage is there an update on this? Has the ready tag since 10/15

[happilymarrieddad]

+1

[octalmage]

@porsager I've got screen capture implemented, but that's it. There's logic in there for the rest but it needs some work. See https://github.com/octalmage/robotjs/tree/bitmap-save.

I don't have the time to work on this currently (and won't for a while), but I'm happy to accept and review PRs.

Thanks!

[carlosgalveias]

While saving the screen capture is not implemented, here's a workaround i used by using the Jimp package:
https://github.com/oliver-moran/jimp

var Jimp = require('jimp');
var robot = require('robotjs');

var screenCaptureToFile = function(path) {
    return new Promise((resolve, reject) => {
        try {
            var picture = robot.screen.capture();
            var image = new Jimp(picture.width, picture.height, function(err, img) {
                img.bitmap.data = picture.image;
                img.getBuffer(Jimp.MIME_PNG, (err, png) => {
                    image.write(path, resolve);
                });
            });
        } catch (e) {
            console.error(e);
            reject(e);
        }
    })
}

screenCaptureToFile('test.png');

[developer239]

Thank you so much.

[omegacoleman]

The solution @carlosgalveias given has made two mistakes, the correct version might be:

    var image = jimp(pic.width, pic.height, function(err, img) {
        img.bitmap.data = pic.image;
        img.scan(0, 0, img.bitmap.width, img.bitmap.height, function (x, y, idx) {
          var red   = img.bitmap.data[ idx + 0 ];
          var blue  = img.bitmap.data[ idx + 2 ];
          img.bitmap.data[ idx + 0 ] = blue;
          img.bitmap.data[ idx + 2 ] = red;
        });
        img.write("./" + n + (new Date().getTime()) + '.png');
    });

but this need improving as well. Jimp and RobotJS pick differently from BGR and RGB, so it needs reversing. Actually I think performing an option in robotjs about RGB and BGR is a better idea.

[johnmarlow007]

Hi guys. I just wanted to say "CONGRATULATIONS!" for this project.
I have tried a few things and it works pretty much flowlessly.

Now I too need this screen capture feature with image saved locally for further processing and I highly encourage your efforts to get it working.

I would like to support the project but will need to get deeper in the study of how its done and I hope I can help with my limited knowledge.
Thank you and best of luck !
Andy

[arfost]

Hi,

I'm not sure why but the two above functions with jimp didn't work for me, and I needed to save the captured picture, so here is my version of the function using JIMP if it can help someone :)

var Jimp = require('jimp');
var robot = require('robotjs');

var screenCaptureToFile = function (path,  robotScreenPic) {
    return new Promise((resolve, reject) => {
        try {
            var image = new Jimp(robotScreenPic.width, robotScreenPic.height);
            image.scan(0, 0, image.bitmap.width, image.bitmap.height, function (x, y, idx) {
                var color = robotScreenPic.colorAt(x, y);
                var red = parseInt(color[0] + color[1], 16);
                var green = parseInt(color[2] + color[3], 16);
                var blue = parseInt(color[4] + color[5], 16);

                image.bitmap.data[idx + 0] = Number(red);
                image.bitmap.data[idx + 1] = Number(green);
                image.bitmap.data[idx + 2] = Number(blue);
                image.bitmap.data[idx + 3] = 255;
            });
            image.write(path, resolve);

        } catch (e) {
            console.error(e);
            reject(e);
        }
    })
}

Thanks to @carlosgalveias and @omegacoleman for the starter, I couldn't find it without them.
Beware, it can be very slow for big pictures

[vernondegoede]

@arfost I experienced the same issue with large pictures (especially retina screens). It took too long to process the picture (about ~600ms), so I ended up using the desktop-screenshot package.

https://github.com/vernondegoede/nonbilight/blob/master/renderer/components/TheaterMode.js#L50

[carlosgalveias]

In the meantime i found this:
http://tipsandtricks.runicsoft.com/Cpp/BitmapTutorial.html#chapter4

Maybe this can be done natively in C++ so that its faster, in certain cases the JIMP solution is taking 2 seconds.
But getting the output buffer as a RGB instead of a BRG is a must for me.
I would implement it, and tried but crashes all over the place (i'm not a c++ programmer)

@octalmage , what do you think ?

[Deltatiger]

This has to be done in C++ to get some speed benefits. Currently the entire screen buffer's memory is copied onto a bitmap object in C++ and then returned to JS. The code snippet posted above iterates over each pixel (I guess) before the image is generated. For a simple 1024 x 768 resolution screen (Which is a very small screen compared to what is actually out there) this is going to run a lot of times.
Instead if RobotJS binary can directly create the specified file in BMP format, it should be way easier and a lot quicker.
And since the data is still being generated as a Bitmap image, creating a BMP file should be easy.

[carlosgalveias]

Ok, i managed to do it in C++ and now its fast.

Made a fork here https://github.com/carlosgalveias/robotjs in windows that outputs the buffer as a RGB and not as a BRG

Added the function to MMBitmap.c :

MMBitmapRef ConvertBMPToRGBBuffer (MMBitmapRef bitmap) {

assert(bitmap != NULL);
uint8_t *rgbBitmap = NULL;

if (bitmap->imageBuffer != NULL) {
	const size_t bufsize = bitmap->height * bitmap->bytewidth;
	rgbBitmap = malloc(bufsize);
	if (rgbBitmap == NULL) return NULL;
	memcpy(rgbBitmap, bitmap->imageBuffer, bufsize);
}

// find the number of padding bytes
int padding = 0;
int scanlinebytes = bitmap->bytewidth;
while ( ( scanlinebytes + padding ) % bitmap->bytewidth != 0 )     // DWORD = 4 bytes
	padding++;
// get the padded scanline bitmap->bytewidth
int psw = scanlinebytes + padding;

// now we loop trough all bytes of the original buffer, 
// swap the R and B bytes and the scanlines
long bufpos = 0;   
long newpos = 0;
for ( int y = 0; y < bitmap->height; y++ )
	for ( int x = 0; x < bitmap->bytesPerPixel * bitmap->width; x+=bitmap->bytesPerPixel )
	{
		newpos = y * bitmap->bytesPerPixel * bitmap->width + x;     
		rgbBitmap[newpos] = bitmap->imageBuffer[newpos + 2];       
		rgbBitmap[newpos + 2] = bitmap->imageBuffer[newpos];     
	}

// Do not forget to destroy the original bitmap if you dont need it to avoid memory leaks
return createMMBitmap(rgbBitmap,
                      bitmap->width,
                      bitmap->height,
                      bitmap->bytewidth,
                      bitmap->bitsPerPixel,
                      bitmap->bytesPerPixel); }

On screengrab.c i added:

/* Copy the data to our pixel buffer. */

if (bitmap != NULL) {
	bitmap->imageBuffer = malloc(bitmap->bytewidth * bitmap->height);
	memcpy(bitmap->imageBuffer, data, bitmap->bytewidth * bitmap->height);
	bitmapRGB = ConvertBMPToRGBBuffer(bitmap);
}
destroyMMBitmap(bitmap);

[Zazck]

@arfost process image as buffer(not as array) will speed up your convertion.

function screenCaptureToFile(robotScreenPic) {
  return new Promise((resolve, reject) => {
    try {
      const image = new Jimp(robotScreenPic.width, robotScreenPic.height);
      let pos = 0;
      image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
        /* eslint-disable no-plusplus */
        image.bitmap.data[idx + 2] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 1] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 0] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 3] = robotScreenPic.image.readUInt8(pos++);
        /* eslint-enable no-plusplus */
      });
      resolve(image);
    } catch (e) {
      console.error(e);
      reject(e);
    }
  });
}

it will be 400 times faster

buffer: 50 convert: 21213
buffer: 55 convert: 21277
buffer: 40 convert: 21823

[jtoy]

is this considered complete?

[ghost]

thanks @Zazck for that snippet.

I was having an issue on my computer with the alpha channel not rendering correctly; it was spitting out large black rectangles over certain areas. In case anyone else is having the same issue, this forces the alpha channel to always be 100% during the conversion:

function screenCaptureToFile(robotScreenPic) {
  return new Promise((resolve, reject) => {
    try {
      const image = new Jimp(robotScreenPic.width, robotScreenPic.height);
      let pos = 0;
      image.scan(0, 0, image.bitmap.width, image.bitmap.height, (x, y, idx) => {
        /* eslint-disable no-plusplus */
        image.bitmap.data[idx + 2] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 1] = robotScreenPic.image.readUInt8(pos++);
        image.bitmap.data[idx + 0] = robotScreenPic.image.readUInt8(pos++);
        pos++
        image.bitmap.data[idx + 3] = 255;
        /* eslint-enable no-plusplus */
      });
      resolve(image);
    } catch (e) {
      console.error(e);
      reject(e);
    }
  });
}

[Sarkell]

@fanfare I didn't come across this issue. And I have been using this solution for a week very intensively

@octalmage It would be wonderful to have this feature ASAP. The solution seems to be found

[will123195]

This worked for me:

const robot = require('robotjs')
const Jimp = require('jimp')

function captureImage({ x, y, w, h }) {
  const pic = robot.screen.capture(x, y, w, h)
  const width = pic.byteWidth / pic.bytesPerPixel // pic.width is sometimes wrong!
  const height = pic.height
  const image = new Jimp(width, height)
  let red, green, blue
  pic.image.forEach((byte, i) => {
    switch (i % 4) {
      case 0: return blue = byte
      case 1: return green = byte
      case 2: return red = byte
      case 3: 
        image.bitmap.data[i - 3] = red
        image.bitmap.data[i - 2] = green
        image.bitmap.data[i - 1] = blue
        image.bitmap.data[i] = 255
    }
  })
  return image
}

captureImage({ x, y, w, h }).write('capture.png')

[sebastiaorealino]

This worked for me:

const robot = require('robotjs')
const Jimp = require('jimp')

function captureImage({ x, y, w, h }) {
  const pic = robot.screen.capture(x, y, w, h)
  const width = pic.byteWidth / pic.bytesPerPixel // pic.width is sometimes wrong!
  const height = pic.height
  const image = new Jimp(width, height)
  let red, green, blue
  pic.image.forEach((byte, i) => {
    switch (i % 4) {
      case 0: return blue = byte
      case 1: return green = byte
      case 2: return red = byte
      case 3: 
        image.bitmap.data[i - 3] = red
        image.bitmap.data[i - 2] = green
        image.bitmap.data[i - 1] = blue
        image.bitmap.data[i] = 255
    }
  })
  return image
}

captureImage({ x, y, w, h }).write('capture.png')

Try to use @Zazck solution.
His code is almost twice as fast:

@will123195 code: 639.065ms
@Zazck code: 371.819ms

Thanks @Zazck!

[TongHaizhang]

Maybe you can speed up the pace of updates

[wangrongding]

期待

[wburgess-around]

If anyone else just needs to grab a screenshot on MacOS you can just do:

await robot.keyTap('3', [ 'command', 'shift' ])

to take a screenshot that will auto-save to your Desktop. Obviously will not support custom sizes but this is all I needed to debug a remote machine.

[Psalm01-bot]

Weldone at octalmage, pls can you pls update on search function

[arthurwolf]

Any progress on this?

Amazing project.

[thenengah]

Would be an amazing feature.

According to chat GPT there is a full javascript solution:

const robot = require('robotjs');
const Jimp = require('jimp');

// Function to capture a screenshot
async function captureScreenshot() {
    // Get screen size
    const screenSize = robot.getScreenSize();

    // Capture the screenshot using RobotJS
    const img = robot.screen.capture(0, 0, screenSize.width, screenSize.height);

    // Create an empty buffer for the image
    const image = new Jimp({
        width: img.width,
        height: img.height,
        data: img.image
    });

    // Save the image to a file
    await image.writeAsync('screenshot.png');
    console.log('Screenshot saved as screenshot.png');
}

// Call the function
captureScreenshot().catch(console.error);

We will implementing this feature internally soon. Probably just start with that code to see if it works, should be cross platform.

[thenengah]

In fact, it looks it's already implemented with: robot.screen.capture. I'll try to report back to see if this actually works.

[thenengah]

It's implemented but in raw pixel data. That's probably enough, but most people will need to do something like the code below. Since it requires picking another node module might be best for people to implement it individually since it's not difficult and probably not everyone would want to use sharp.

const path = require("node:path"):
const robot = require("@jitsi/robotjs");
const sharp = require("sharp");

/*
 * Convert raw image to something that's easier to work with 
 */
const getSharpImage = async () => {
  const screenSize = robot.getScreenSize();
  const rawImage = robot.screen.capture(
    0,
    0,
    screenSize.width,
    screenSize.height
  );
  const rgbaBuffer = Buffer.from(rawImage.image);
  for (let i = 0; i < rawImage.image.length; i += 4) {
    rgbaBuffer[i + 0] = rawImage.image[i + 2];
    rgbaBuffer[i + 1] = rawImage.image[i + 1];
    rgbaBuffer[i + 2] = rawImage.image[i + 0];
    rgbaBuffer[i + 3] = rawImage.image[i + 3];
  }
  return sharp(rgbaBuffer, {
    raw: {
      width: rawImage.width,
      height: rawImage.height,
      channels: 4,
    },
  })
};

/*
 * Example of how to use getSharpImage
 */
(async () => {
  try {
    const sharpImage = await getSharpImage();
    const filePath = path.join(__dirname, `screenshot-${Date.now()}.png`);
    await sharpImage.toFile(filePath);
  } catch (err) {
    console.error(err)
  }
})();

[porsager]

I recently made this for fairly efficient cross platform screenshots - don't know if it would fit in?

https://github.com/porsager/screendump