diff --git a/egui/src/context.rs b/egui/src/context.rs
index edd676bd..3d6d162b 100644
--- a/egui/src/context.rs
+++ b/egui/src/context.rs
@@ -223,6 +223,17 @@ impl Context {
         if !same_as_current {
             self.fonts = Some(Arc::new(Fonts::from_definitions(font_definitions)));
         }
+
+        // Ensure we register the background area so panels and background ui can catch clicks:
+        let screen_rect = self.input.screen_rect();
+        self.memory().areas.set_state(
+            LayerId::background(),
+            containers::area::State {
+                pos: screen_rect.min,
+                size: screen_rect.size(),
+                interactable: true,
+            },
+        );
     }
 
     /// Call at the end of each frame.
@@ -268,17 +279,6 @@ impl Context {
     fn fullscreen_ui(self: &Arc<Self>) -> Ui {
         let rect = self.input.screen_rect();
         let layer_id = LayerId::background();
-        // Ensure we register the background area so it is painted
-        // and so we handle interactions properly (clicks on it etc).
-        // TODO: we shouldn't need to register areas. Maybe GraphicLayers should take care of it?
-        self.memory().areas.set_state(
-            layer_id,
-            containers::area::State {
-                pos: rect.min,
-                size: rect.size(),
-                interactable: true,
-            },
-        );
         Ui::new(self.clone(), layer_id, layer_id.id, rect, rect)
     }
 
diff --git a/egui/src/layers.rs b/egui/src/layers.rs
index 0446bbfa..ba983959 100644
--- a/egui/src/layers.rs
+++ b/egui/src/layers.rs
@@ -17,6 +17,16 @@ pub enum Order {
     /// Debug layer, always painted last / on top
     Debug,
 }
+impl Order {
+    const COUNT: usize = 5;
+    const ALL: [Order; Self::COUNT] = [
+        Self::Background,
+        Self::Middle,
+        Self::Foreground,
+        Self::Tooltip,
+        Self::Debug,
+    ];
+}
 
 /// An identifier for a paint layer.
 /// Also acts as an identifier for `Area`:s.
@@ -52,6 +62,10 @@ pub struct PaintCmdIdx(usize);
 pub struct PaintList(Vec<(Rect, PaintCmd)>);
 
 impl PaintList {
+    pub fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+
     /// Returns the index of the new command that can be used with `PaintList::set`.
     pub fn add(&mut self, clip_rect: Rect, cmd: PaintCmd) -> PaintCmdIdx {
         let idx = PaintCmdIdx(self.0.len());
@@ -76,13 +90,14 @@ impl PaintList {
     }
 }
 
-// TODO: improve GraphicLayers
 #[derive(Clone, Default)]
-pub struct GraphicLayers(AHashMap<LayerId, PaintList>);
+pub struct GraphicLayers([AHashMap<Id, PaintList>; Order::COUNT]);
 
 impl GraphicLayers {
     pub fn list(&mut self, layer_id: LayerId) -> &mut PaintList {
-        self.0.entry(layer_id).or_default()
+        self.0[layer_id.order as usize]
+            .entry(layer_id.id)
+            .or_default()
     }
 
     pub fn drain(
@@ -91,16 +106,29 @@ impl GraphicLayers {
     ) -> impl ExactSizeIterator<Item = (Rect, PaintCmd)> {
         let mut all_commands: Vec<_> = Default::default();
 
-        for layer_id in area_order {
-            if let Some(commands) = self.0.get_mut(layer_id) {
+        for &order in &Order::ALL {
+            let order_map = &mut self.0[order as usize];
+
+            // If a layer is empty at the start of the frame
+            // the nobody has added to it, and it is old and defunct.
+            // Free it to save memory:
+            order_map.retain(|_, list| !list.is_empty());
+
+            // First do the layers part of area_order:
+            for layer_id in area_order {
+                if layer_id.order == order {
+                    if let Some(commands) = order_map.get_mut(&layer_id.id) {
+                        all_commands.extend(commands.0.drain(..));
+                    }
+                }
+            }
+
+            // Also draw areas that are missing in `area_order`:
+            for commands in order_map.values_mut() {
                 all_commands.extend(commands.0.drain(..));
             }
         }
 
-        if let Some(commands) = self.0.get_mut(&LayerId::debug()) {
-            all_commands.extend(commands.0.drain(..));
-        }
-
         all_commands.into_iter()
     }
 }
diff --git a/egui/src/memory.rs b/egui/src/memory.rs
index c952e7d5..6c5cefeb 100644
--- a/egui/src/memory.rs
+++ b/egui/src/memory.rs
@@ -113,6 +113,8 @@ impl Interaction {
     }
 }
 
+/// Keeps track of `Area`s, which are free-floating `Ui`s.
+/// These `Area`s can be in any `Order`.
 #[derive(Clone, Debug, Default)]
 #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))]
 #[cfg_attr(feature = "serde", serde(default))]