migrate: convert __unmap_and_move() to use folios

+37 -38

1 changed file

expand all

migrate.c

+37 -38

mm/migrate.c

··· 993 993 return rc; 994 994 } 995 995 996 - static int __unmap_and_move(struct page *page, struct page *newpage, 996 + static int __unmap_and_move(struct folio *src, struct folio *dst, 997 997 int force, enum migrate_mode mode) 998 998 { 999 - struct folio *folio = page_folio(page); 1000 - struct folio *dst = page_folio(newpage); 1001 999 int rc = -EAGAIN; 1002 1000 bool page_was_mapped = false; 1003 1001 struct anon_vma *anon_vma = NULL; 1004 - bool is_lru = !__PageMovable(page); 1002 + bool is_lru = !__PageMovable(&src->page); 1005 1003 1006 - if (!trylock_page(page)) { 1004 + if (!folio_trylock(src)) { 1007 1005 if (!force || mode == MIGRATE_ASYNC) 1008 1006 goto out; 1009 1007 ··· 1021 1023 if (current->flags & PF_MEMALLOC) 1022 1024 goto out; 1023 1025 1024 - lock_page(page); 1026 + folio_lock(src); 1025 1027 } 1026 1028 1027 - if (PageWriteback(page)) { 1029 + if (folio_test_writeback(src)) { 1028 1030 /* 1029 1031 * Only in the case of a full synchronous migration is it 1030 1032 * necessary to wait for PageWriteback. In the async case, ··· 1041 1043 } 1042 1044 if (!force) 1043 1045 goto out_unlock; 1044 - wait_on_page_writeback(page); 1046 + folio_wait_writeback(src); 1045 1047 } 1046 1048 1047 1049 /* 1048 - * By try_to_migrate(), page->mapcount goes down to 0 here. In this case, 1049 - * we cannot notice that anon_vma is freed while we migrates a page. 1050 + * By try_to_migrate(), src->mapcount goes down to 0 here. In this case, 1051 + * we cannot notice that anon_vma is freed while we migrate a page. 1050 1052 * This get_anon_vma() delays freeing anon_vma pointer until the end 1051 1053 * of migration. File cache pages are no problem because of page_lock() 1052 1054 * File Caches may use write_page() or lock_page() in migration, then, ··· 1058 1060 * because that implies that the anon page is no longer mapped 1059 1061 * (and cannot be remapped so long as we hold the page lock). 1060 1062 */ 1061 - if (PageAnon(page) && !PageKsm(page)) 1062 - anon_vma = page_get_anon_vma(page); 1063 + if (folio_test_anon(src) && !folio_test_ksm(src)) 1064 + anon_vma = page_get_anon_vma(&src->page); 1063 1065 1064 1066 /* 1065 1067 * Block others from accessing the new page when we get around to 1066 1068 * establishing additional references. We are usually the only one 1067 - * holding a reference to newpage at this point. We used to have a BUG 1068 - * here if trylock_page(newpage) fails, but would like to allow for 1069 - * cases where there might be a race with the previous use of newpage. 1069 + * holding a reference to dst at this point. We used to have a BUG 1070 + * here if folio_trylock(dst) fails, but would like to allow for 1071 + * cases where there might be a race with the previous use of dst. 1070 1072 * This is much like races on refcount of oldpage: just don't BUG(). 1071 1073 */ 1072 - if (unlikely(!trylock_page(newpage))) 1074 + if (unlikely(!folio_trylock(dst))) 1073 1075 goto out_unlock; 1074 1076 1075 1077 if (unlikely(!is_lru)) { 1076 - rc = move_to_new_folio(dst, folio, mode); 1078 + rc = move_to_new_folio(dst, src, mode); 1077 1079 goto out_unlock_both; 1078 1080 } 1079 1081 ··· 1081 1083 * Corner case handling: 1082 1084 * 1. When a new swap-cache page is read into, it is added to the LRU 1083 1085 * and treated as swapcache but it has no rmap yet. 1084 - * Calling try_to_unmap() against a page->mapping==NULL page will 1086 + * Calling try_to_unmap() against a src->mapping==NULL page will 1085 1087 * trigger a BUG. So handle it here. 1086 1088 * 2. An orphaned page (see truncate_cleanup_page) might have 1087 1089 * fs-private metadata. The page can be picked up due to memory ··· 1089 1091 * invisible to the vm, so the page can not be migrated. So try to 1090 1092 * free the metadata, so the page can be freed. 1091 1093 */ 1092 - if (!page->mapping) { 1093 - VM_BUG_ON_PAGE(PageAnon(page), page); 1094 - if (page_has_private(page)) { 1095 - try_to_free_buffers(folio); 1094 + if (!src->mapping) { 1095 + if (folio_test_private(src)) { 1096 + try_to_free_buffers(src); 1096 1097 goto out_unlock_both; 1097 1098 } 1098 - } else if (page_mapped(page)) { 1099 + } else if (folio_mapped(src)) { 1099 1100 /* Establish migration ptes */ 1100 - VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma, 1101 - page); 1102 - try_to_migrate(folio, 0); 1101 + VM_BUG_ON_FOLIO(folio_test_anon(src) && 1102 + !folio_test_ksm(src) && !anon_vma, src); 1103 + try_to_migrate(src, 0); 1103 1104 page_was_mapped = true; 1104 1105 } 1105 1106 1106 - if (!page_mapped(page)) 1107 - rc = move_to_new_folio(dst, folio, mode); 1107 + if (!folio_mapped(src)) 1108 + rc = move_to_new_folio(dst, src, mode); 1108 1109 1109 1110 /* 1110 - * When successful, push newpage to LRU immediately: so that if it 1111 + * When successful, push dst to LRU immediately: so that if it 1111 1112 * turns out to be an mlocked page, remove_migration_ptes() will 1112 - * automatically build up the correct newpage->mlock_count for it. 1113 + * automatically build up the correct dst->mlock_count for it. 1113 1114 * 1114 1115 * We would like to do something similar for the old page, when 1115 1116 * unsuccessful, and other cases when a page has been temporarily 1116 1117 * isolated from the unevictable LRU: but this case is the easiest. 1117 1118 */ 1118 1119 if (rc == MIGRATEPAGE_SUCCESS) { 1119 - lru_cache_add(newpage); 1120 + folio_add_lru(dst); 1120 1121 if (page_was_mapped) 1121 1122 lru_add_drain(); 1122 1123 } 1123 1124 1124 1125 if (page_was_mapped) 1125 - remove_migration_ptes(folio, 1126 - rc == MIGRATEPAGE_SUCCESS ? dst : folio, false); 1126 + remove_migration_ptes(src, 1127 + rc == MIGRATEPAGE_SUCCESS ? dst : src, false); 1127 1128 1128 1129 out_unlock_both: 1129 - unlock_page(newpage); 1130 + folio_unlock(dst); 1130 1131 out_unlock: 1131 1132 /* Drop an anon_vma reference if we took one */ 1132 1133 if (anon_vma) 1133 1134 put_anon_vma(anon_vma); 1134 - unlock_page(page); 1135 + folio_unlock(src); 1135 1136 out: 1136 1137 /* 1137 - * If migration is successful, decrease refcount of the newpage, 1138 + * If migration is successful, decrease refcount of dst, 1138 1139 * which will not free the page because new page owner increased 1139 1140 * refcounter. 1140 1141 */ 1141 1142 if (rc == MIGRATEPAGE_SUCCESS) 1142 - put_page(newpage); 1143 + folio_put(dst); 1143 1144 1144 1145 return rc; 1145 1146 } ··· 1154 1157 enum migrate_reason reason, 1155 1158 struct list_head *ret) 1156 1159 { 1160 + struct folio *dst, *src = page_folio(page); 1157 1161 int rc = MIGRATEPAGE_SUCCESS; 1158 1162 struct page *newpage = NULL; 1159 1163 ··· 1172 1174 newpage = get_new_page(page, private); 1173 1175 if (!newpage) 1174 1176 return -ENOMEM; 1177 + dst = page_folio(newpage); 1175 1178 1176 1179 newpage->private = 0; 1177 - rc = __unmap_and_move(page, newpage, force, mode); 1180 + rc = __unmap_and_move(src, dst, force, mode); 1178 1181 if (rc == MIGRATEPAGE_SUCCESS) 1179 1182 set_page_owner_migrate_reason(newpage, reason); 1180 1183

Configure Feed

Configure Feed