diff --git a/servers/visual/rasterizer_rd/rasterizer_scene_high_end_rd.cpp b/servers/visual/rasterizer_rd/rasterizer_scene_high_end_rd.cpp
index c0aba2658e4..e3c3cdf0bf6 100644
--- a/servers/visual/rasterizer_rd/rasterizer_scene_high_end_rd.cpp
+++ b/servers/visual/rasterizer_rd/rasterizer_scene_high_end_rd.cpp
@@ -2690,6 +2690,8 @@ RasterizerSceneHighEndRD::RasterizerSceneHighEndRD(RasterizerStorageRD *p_storag
 }
 
 RasterizerSceneHighEndRD::~RasterizerSceneHighEndRD() {
+	directional_shadow_atlas_set_size(0);
+
 	//clear base uniform set if still valid
 	if (view_dependant_uniform_set.is_valid() && RD::get_singleton()->uniform_set_is_valid(view_dependant_uniform_set)) {
 		RD::get_singleton()->free(view_dependant_uniform_set);
diff --git a/servers/visual/rasterizer_rd/rasterizer_scene_rd.cpp b/servers/visual/rasterizer_rd/rasterizer_scene_rd.cpp
index e95b7c0b2ad..317ffb994f4 100644
--- a/servers/visual/rasterizer_rd/rasterizer_scene_rd.cpp
+++ b/servers/visual/rasterizer_rd/rasterizer_scene_rd.cpp
@@ -3125,8 +3125,6 @@ RasterizerSceneRD::RasterizerSceneRD(RasterizerStorageRD *p_storage) {
 }
 
 RasterizerSceneRD::~RasterizerSceneRD() {
-	directional_shadow_atlas_set_size(0);
-
 	for (Map<Vector2i, ShadowMap>::Element *E = shadow_maps.front(); E; E = E->next()) {
 		RD::get_singleton()->free(E->get().depth);
 	}