InjectSyntheticPointerInput не работает должным образом, хотя NUnitTest проходит ⇐ C#

[Anonymous]

Я пытаюсь смоделировать свойства стилуса, такие как координаты x, y, наклон и давление, которые я записал в файл json. В NUnitTest я пытаюсь смоделировать точно такой же json с помощью WinAppDriver в SamsungNotes.
Модуль PenInjector

Код: Выделить всё

using SamsungNotesAutomation.Models;
using SamsungNotesAutomation.PInvoke;
using System.Diagnostics;

namespace SamsungNotesAutomation.Helpers
{
public class PenInjector : IDisposable
{
private const uint MAX_PRESSURE = 1024;
private const int TILT_RANGE = 90; // -90 to +90 degrees
private readonly IntPtr _penDevice;
private uint _pointerId = 1;

public PenInjector()
{
_penDevice = NativeMethods.CreateSyntheticPointerDevice(
POINTER_INPUT_TYPE.PEN,
1,
NativeMethods.POINTER_FEEDBACK_DEFAULT);

if (_penDevice == IntPtr.Zero)
{
int err = NativeMethods.GetLastError();
throw new Exception($"CreateSyntheticPointerDevice(PEN) failed. Win32 error: {err}");
}

Debug.WriteLine("PenInjector: Synthetic pen device created successfully");
}

public void Dispose()
{
if (_penDevice != IntPtr.Zero)
{
NativeMethods.DestroySyntheticPointerDevice(_penDevice);
Debug.WriteLine("PenInjector: Synthetic pen device destroyed");
}
GC.SuppressFinalize(this);
}

~PenInjector()
{
Dispose();
}

/// 
/// Inject a full stroke.  Assumes stroke.Points already contain screen coordinates.
/// 
public bool InjectStroke(StrokeData stroke)
{
if (stroke == null || stroke.Points == null || stroke.Points.Count == 0)
{
Debug.WriteLine("InjectStroke: no points");
return false;
}

Debug.WriteLine($"InjectStroke: Starting injection of {stroke.Points.Count} points for character '{stroke.Character}'");

DateTime? lastTs = null;

for (int i = 0; i < stroke.Points.Count; i++)
{
var p = stroke.Points[i];

//Timing based on original timestamps(clamped)
if (lastTs.HasValue)
{
double dt = (p.Timestamp - lastTs.Value).TotalMilliseconds;
if (dt > 0 && dt < 200)
Thread.Sleep((int)dt);
else if (dt >= 200)
Thread.Sleep(5); // Cap at 5ms if timestamps are too long
}

POINTER_FLAGS flags;
if (i == 0)
{
flags = POINTER_FLAGS.DOWN | POINTER_FLAGS.INRANGE | POINTER_FLAGS.INCONTACT | POINTER_FLAGS.PRIMARY;
Debug.WriteLine($"  Point[{i}]: DOWN at ({p.X:F1}, {p.Y:F1}), pressure={p.Pressure:F3}");
}
else if (i == stroke.Points.Count - 1)
{
flags = POINTER_FLAGS.UP | POINTER_FLAGS.INRANGE;
Debug.WriteLine($"  Point[{i}]: UP at ({p.X:F1}, {p.Y:F1}), pressure={p.Pressure:F3}");
}
else
{
flags = POINTER_FLAGS.UPDATE | POINTER_FLAGS.INRANGE | POINTER_FLAGS.INCONTACT;
if (i % 50 == 0) // Log every 50th point to avoid spam
Debug.WriteLine($"   Point[{i}]: UPDATE at ({p.X:F1}, {p.Y:F1}), pressure={p.Pressure:F3}");
}

bool ok = InjectPenPoint((int)Math.Round(p.X),
(int)Math.Round(p.Y),
p.Pressure,
p.TiltX,
p.TiltY,
flags,
p.IsBarrelButtonPressed);

if (!ok)
{
Debug.WriteLine($"InjectStroke: FAILED at point {i}");
return false;
}

lastTs = p.Timestamp;
}

Debug.WriteLine($"InjectStroke: Successfully injected all {stroke.Points.Count} points");
return true;
}

/// 
/// Inject a single pen point with full fidelity (position, pressure, tilt).
/// 
private bool InjectPenPoint(
int x,
int y,
double pressure,
int tiltX,
int tiltY,
POINTER_FLAGS flags,
bool barrelPressed)
{
IntPtr hwnd = NativeMethods.GetForegroundWindow();
// Normalize tilt values from your pen's range (0-4096, center at 2048)
// to Windows expected range (-90 to +90 degrees)
int normalizedTiltX = NormalizeTilt(tiltX);
int normalizedTiltY = NormalizeTilt(tiltY);

var info = new POINTER_PEN_INFO
{
pointerInfo = new POINTER_INFO
{
pointerType = POINTER_INPUT_TYPE.PEN,
pointerId = _pointerId,
pointerFlags = flags,
ptPixelLocation = new POINT(x, y),
dwTime = (uint)Environment.TickCount,
sourceDevice = _penDevice,
hwndTarget = hwnd
},
penFlags = barrelPressed ? PEN_FLAGS.BARREL : PEN_FLAGS.NONE,
penMask = PEN_MASK.PRESSURE | PEN_MASK.TILT_X | PEN_MASK.TILT_Y,
pressure = NormalizePressure(pressure),
rotation = 0,
tiltX = normalizedTiltX,
tiltY = normalizedTiltY
};

bool result = NativeMethods.InjectSyntheticPointerInput(
_penDevice,
new[] { info },
1);

if (!result)
{
int err = NativeMethods.GetLastError();
Debug.WriteLine($"InjectPenPoint FAILED at ({x}, {y}): Win32 error {err}");
return false;
}

return true;
}

/// 
/// Normalize pressure from 0.0-1.0 to 0-1024 for Windows API.
/// 
private uint NormalizePressure(double p)
{
if (p < 0) p = 0;
if (p > 1) p = 1;
return (uint)(p * MAX_PRESSURE);
}

/// 
/// Normalize tilt from your S Pen's range (0-4096, centered at ~2048)
/// to Windows expected range (-90 to +90 degrees).
///
/// Your pen reports: tiltX ~2100, tiltY ~1500
/// Center is around 2048, full range is 0-4096
/// 
private int NormalizeTilt(int rawTilt)
{
// S Pen digitizer range (adjust if your values are different)
const int CENTER = 2048;
const int MAX_RANGE = 2048; // Distance from center to edge

// Convert to normalized range: -1.0 to +1.0
double normalized = (double)(rawTilt - CENTER) / MAX_RANGE;

// Clamp to valid range
if (normalized < -1.0) normalized = -1.0;
if (normalized > 1.0) normalized = 1.0;

// Map to degrees: -90 to +90
int degrees = (int)Math.Round(normalized * TILT_RANGE);

// Final clamp for safety
if (degrees < -TILT_RANGE) degrees = -TILT_RANGE;
if (degrees > TILT_RANGE) degrees = TILT_RANGE;

return degrees;
}

/// 
/// Inject multiple strokes one after another.
/// 
public bool InjectMultipleStrokes(
IList  strokes,
int delayBetweenStrokesMs = 100)
{
if (strokes == null || strokes.Count == 0)
{
Debug.WriteLine("InjectMultipleStrokes: no strokes provided");
return false;
}

Debug.WriteLine($"InjectMultipleStrokes: Injecting {strokes.Count} strokes");

for (int i = 0; i < strokes.Count; i++)
{
Debug.WriteLine($"=== Stroke {i + 1}/{strokes.Count} ===");

if (!InjectStroke(strokes[i]))
{
Debug.WriteLine($"InjectMultipleStrokes: FAILED at stroke {i + 1}");
return false;
}

if (delayBetweenStrokesMs > 0 && i < strokes.Count - 1)
{
Thread.Sleep(delayBetweenStrokesMs);
}
}

Debug.WriteLine("InjectMultipleStrokes: All strokes injected successfully");
return true;
}
}
}

Тестовый пример:

Код: Выделить всё

[Test]
[Order(1)]
public void Test_01_InjectSingleStroke()
{
Debug.WriteLine("\n*** Test_01_InjectSingleStroke START ***");

// Step 1: Create new note
Debug.WriteLine("Step 1: Looking for CreateNoteButton...");
try
{
var newNoteButton = _driver.FindElementByAccessibilityId("CreateNoteButton");
Debug.WriteLine("  Found CreateNoteButton, clicking...");
newNoteButton.Click();
Thread.Sleep(1000);
Debug.WriteLine("  Clicked successfully");
}
catch (Exception ex)
{
Debug.WriteLine($"  ERROR: Could not find CreateNoteButton: {ex.Message}");
Assert.Inconclusive($"Could not find CreateNoteButton. Error: {ex.Message}");
return;
}

// Step 2: Activate pen mode
Debug.WriteLine("Step 2: Looking for Pen mode button...");
try
{
var penMode = _driver.FindElementByName("Pen mode");
Debug.WriteLine("  Found Pen mode, clicking...");
penMode.Click();
Thread.Sleep(500);
Debug.WriteLine("  Pen mode activated");
}
catch (Exception ex)
{
Debug.WriteLine($"  Note: Pen mode button not found (might already be active): {ex.Message}");
}

// Step 3: Find and click canvas
Debug.WriteLine("Step 3: Looking for canvas element...");
WindowsElement canvas;
try
{
canvas = _driver.FindElementByXPath("//*[@AutomationId='SpenComposerViewScrollViewer' and @FrameworkId='XAML' and @ClassName='ScrollViewer']");
Debug.WriteLine("  Found 'Editing' canvas");

// Click canvas to ensure it has focus
canvas.Click();
Thread.Sleep(500);
Debug.WriteLine("  Canvas clicked and should have focus");
}
catch (Exception ex)
{
Debug.WriteLine($"  ERROR: Could not find canvas: {ex.Message}");
Assert.Inconclusive($"Could not find canvas element.  Error: {ex.Message}");
return;
}

// Step 4: Get canvas coordinates
var canvasRect = CoordinateHelper.GetElementScreenRect(canvas);
Debug.WriteLine($"Step 4: Canvas rect: X={canvasRect.X}, Y={canvasRect.Y}, " +
$"W={canvasRect.Width}, H={canvasRect.Height}");

// Step 5: Prepare stroke
var stroke = _testStrokes[0];
Debug.WriteLine($"Step 5: Preparing stroke for character '{stroke.Character}'");
Debug.WriteLine($"  Original stroke bbox: minX={stroke.BoundingBox.MinX:F1}, maxX={stroke.BoundingBox.MaxX:F1}, " +
$"minY={stroke.BoundingBox.MinY:F1}, maxY={stroke.BoundingBox.MaxY:F1}");
Debug.WriteLine($"  Points: {stroke.PointCount}, Avg pressure: {stroke.AveragePressure:F3}");

// Step 6: Transform coordinates
Debug.WriteLine("Step 6: Transforming coordinates...");
var adjustedStroke = CoordinateHelper.AdjustStrokeToScreen(
stroke,
canvasRect,
AppConfig.ScaleStrokesToFit);

Debug.WriteLine($"  Transformed {adjustedStroke.Points.Count} points");
Debug.WriteLine("  First 5 adjusted points:");
for (int i = 0; i < Math.Min(5, adjustedStroke.Points.Count); i++)
{
var p = adjustedStroke.Points[i];
Debug.WriteLine($"    [{i}] = ({p.X:F1}, {p.Y:F1}), pressure={p.Pressure:F3}, " +
$"tilt=({p.TiltX}, {p.TiltY})");
}

// Step 7: Inject stroke
Debug.WriteLine("Step 7: Injecting pen stroke...");
bool success = _penInjector.InjectStroke(adjustedStroke);

if (!success)
{
Debug.WriteLine("  FAILED to inject stroke!");
Assert.Fail("Failed to inject pen stroke");
}

Debug.WriteLine("  SUCCESS! Stroke injected");

// Give time for ink to render
Thread.Sleep(1000);

// Optional: Take screenshot
try
{
var screenshot = _driver.GetScreenshot();
string filename = $"stroke_test_{DateTime.Now:yyyyMMdd_HHmmss}.png";
screenshot.SaveAsFile(filename);
Debug.WriteLine($"  Screenshot saved: {filename}");
}
catch (Exception ex)
{
Debug.WriteLine($"  Could not save screenshot: {ex.Message}");
}

Debug.WriteLine("*** Test_01_InjectSingleStroke END - PASSED ***\n");
}

Проблема, с которой я столкнулся:

После нажатия на холст кажется, что курсор перемещается в верхний левый угол и, вероятно, делает несколько щелчков (я в этом не уверен), но символ, который должен был быть написан, не моделируется на холсте. Что еще более важно, тестовый пример пройден!!! Точные координаты я сделал с помощью mouse_event, он успешно имитирует обводку. Но не работает с InjectSyntheticPointerInput.

Подробнее здесь: https://stackoverflow.com/questions/798 ... est-passes