Not really a geometric problem.
The geometric part was solve by you. If you would look at internal polygon class from fabricjs you would notice that polygon as a calcDimension function where every point gets an offset:
http://fabricjs.com/docs/fabric.Polygon.html
To calculate canvas position you have to add that offset back before transforming.
var canvas = new fabric.Canvas("c", {selection: false});
var polygon = new fabric.Polygon([
new fabric.Point(200, 50),
new fabric.Point(250, 150),
new fabric.Point(150, 150)
]);
polygon.on("modified", function () {
var matrix = this.calcTransformMatrix();
var transformedPoints = this.get("points")
.map(function(p){
return new fabric.Point(p.x - polygon.minX -polygon.width/2, p.y - polygon.minY - polygon.height/2);
})
.map(function(p){
return fabric.util.transformPoint(p, matrix);
});
var circles = transformedPoints.map(function(p){
return new fabric.Circle({
left: p.x,
top: p.y,
radius: 3,
fill: "red",
originX: "center",
originY: "center",
hasControls: false,
hasBorders: false,
selectable: false
});
});
this.canvas.clear().add(this).add.apply(this.canvas, circles).setActiveObject(this).renderAll();
});
canvas.add(polygon).renderAll();canvas {
border: 1px solid;
}<script src="https://cdnjs.cloudflare.com/ajax/libs/fabric.js/1.6.3/fabric.js"></script>
<p>
Move, scale and rotate the polygon. The three red dots should match with the corners of the polygon after each modification.
</p>
<canvas id="c" width="600" height="400"></canvas>