wayland/platformmenu: fix flipped positions and submenu y positions

src/core/platformmenu.cpp

@@ -104,6 +104,8 @@ bool PlatformMenuEntry::display(QObject* parentWindow, int relativeX, int relati
 	this->qmenu->createWinId();
 	this->qmenu->windowHandle()->setTransientParent(window);
 
+	// Skips screen edge repositioning so it can be left to the compositor on wayland.
+	this->qmenu->targetPosition = point;
 	this->qmenu->popup(point);
 
 	return true;

src/core/platformmenu.hpp

@@ -26,6 +26,7 @@ public:
 	void setVisible(bool visible) override;
 
 	PlatformMenuQMenu* containingMenu = nullptr;
+	QPoint targetPosition;
 };
 
 class PlatformMenuEntry: public QObject {

src/wayland/platformmenu.cpp

@@ -2,31 +2,60 @@
 
 #include <private/qwayland-xdg-shell.h>
 #include <qmargins.h>
+#include <qnamespace.h>
+#include <qsize.h>
+#include <qvariant.h>
 #include <qwindow.h>
 
 #include "../core/platformmenu.hpp"
 
 using namespace qs::menu::platform;
-using namespace QtWayland;
 
 // fixes positioning of submenus when hitting screen edges
 void platformMenuHook(PlatformMenuQMenu* menu) {
 	auto* window = menu->windowHandle();
 
 	auto constraintAdjustment = QtWayland::xdg_positioner::constraint_adjustment_flip_x
-	                          | QtWayland::xdg_positioner::constraint_adjustment_flip_y;
+	                          | QtWayland::xdg_positioner::constraint_adjustment_flip_y
+	                          | QtWayland::xdg_positioner::constraint_adjustment_slide_x
+	                          | QtWayland::xdg_positioner::constraint_adjustment_slide_y
+	                          | QtWayland::xdg_positioner::constraint_adjustment_resize_x
+	                          | QtWayland::xdg_positioner::constraint_adjustment_resize_y;
 
 	window->setProperty("_q_waylandPopupConstraintAdjustment", constraintAdjustment);
 
+	Qt::Edges anchor;
+	Qt::Edges gravity;
+
 	if (auto* containingMenu = menu->containingMenu) {
 		auto geom = containingMenu->actionGeometry(menu->menuAction());
 
+		// Forces action rects to be refreshed. Without this the geometry is intermittently null.
+		containingMenu->sizeHint();
+
 		// use the first action to find the offsets relative to the containing window
 		auto baseGeom = containingMenu->actionGeometry(containingMenu->actions().first());
 		geom += QMargins(0, baseGeom.top(), 0, baseGeom.top());
 
 		window->setProperty("_q_waylandPopupAnchorRect", geom);
+
+		auto sideEdge = menu->isRightToLeft() ? Qt::LeftEdge : Qt::RightEdge;
+		anchor = Qt::TopEdge | sideEdge;
+		gravity = Qt::BottomEdge | sideEdge;
+	} else if (auto* parent = window->transientParent()) {
+		// The menu geometry will be adjusted to flip internally by qt already, but it ends up off by
+		// one pixel which causes the compositor to also flip which results in the menu being placed
+		// left of the edge by its own width. To work around this the intended position is stored prior
+		// to tampering by qt.
+		auto anchorRect = QRect(menu->targetPosition - parent->geometry().topLeft(), QSize(1, 1));
+		window->setProperty("_q_waylandPopupAnchorRect", anchorRect);
+
+		anchor = Qt::BottomEdge | Qt::RightEdge;
+		gravity = Qt::BottomEdge | Qt::RightEdge;
 	}
+
+	window->setProperty("_q_waylandPopupAnchor", QVariant::fromValue(anchor));
+	window->setProperty("_q_waylandPopupGravity", QVariant::fromValue(gravity));
 }
 
 void installPlatformMenuHook() { PlatformMenuEntry::registerCreationHook(&platformMenuHook); }