Last summer I made some more progress not documented here an I was even able to reproduce the behaviour cited below. Still more complex combination of rotation and translation where still wrong.
Now I am trying to get an application working that has views rotated by 90 degrees and this completely fails.
After experimenting on my Apple I came up with code that works somewhat better, but it still isn't completely correct. It looks like the code would be simpler when implementing setBoundsOrigin: based on translateOriginToPoint:.
I put the code here for future reference:
 (void) setBoundsOrigin: (NSPoint)newOrigin
{
if (_is_rotated_from_base)
{
NSAffineTransform *matrix;
NSRect frame = _frame;
NSPoint offset;
frame.origin = NSMakePoint(0, 0);
if (_coordinates_valid)
{
(*invalidateImp)(self, invalidateSel);
}
if (_boundsMatrix == nil)
{
_boundsMatrix = [NSAffineTransform new];
offset = NSMakePoint(NSMinX(_bounds), NSMinY(_bounds));
}
else
{
matrix = [_boundsMatrix copy];
[matrix invert];
offset = [matrix transformPoint: NSMakePoint(0, 0)];
RELEASE(matrix);
}
NSLog(@"Translate by %g, %g", newOrigin.x + offset.x, newOrigin.y + offset.y);
[_boundsMatrix translateXBy: newOrigin.x + offset.x
yBy: newOrigin.y + offset.y];
// [_boundsMatrix setFrameOrigin: NSMakePoint(newOrigin.x, newOrigin.y)];
// Adjust bounds
matrix = [_boundsMatrix copy];
[matrix invert];
[matrix boundingRectFor: frame result: &_bounds];
RELEASE(matrix);
[self resetCursorRects];
if (_post_bounds_changes)
{
[nc postNotificationName: NSViewBoundsDidChangeNotification
object: self];
}
}
else if (NSEqualPoints(_bounds.origin, newOrigin) == NO)
{
if (_coordinates_valid)
{
(*invalidateImp)(self, invalidateSel);
}
_bounds.origin = newOrigin;
if (_boundsMatrix == nil)
{
_boundsMatrix = [NSAffineTransform new];
}
[_boundsMatrix setFrameOrigin: NSMakePoint(newOrigin.x, newOrigin.y)];
[self resetCursorRects];
if (_post_bounds_changes)
{
[nc postNotificationName: NSViewBoundsDidChangeNotification
object: self];
}
}
}
 (void) translateOriginToPoint: (NSPoint)point
{
NSPoint offset;
if (_boundsMatrix == nil)
{
offset = NSMakePoint(NSMinX(_bounds), NSMinY(_bounds));
}
else
{
NSAffineTransform *matrix;
NSRect frame = _frame;
frame.origin = NSMakePoint(0, 0);
matrix = [_boundsMatrix copy];
[matrix invert];
offset = [matrix transformPoint: NSMakePoint(0, 0)];
RELEASE(matrix);
}
[self setBoundsOrigin: NSMakePoint(offset.x  point.x,
offset.y  point.y)];
// [self setBoundsOrigin: NSMakePoint(NSMinX(_bounds)  point.x,
// NSMinY(_bounds)  point.y)];
}
