From 0134ed4e041583da4680d4956299681d20d6ec60 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Wed, 14 Feb 2024 16:01:06 +0000 Subject: [PATCH 01/44] work in progress --- firedrake/linear_solver.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/firedrake/linear_solver.py b/firedrake/linear_solver.py index 8f450d6c46..4a618c8d38 100644 --- a/firedrake/linear_solver.py +++ b/firedrake/linear_solver.py @@ -181,8 +181,20 @@ def solve(self, x, b): else: acc = x.dat.vec_wo - with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): - self.ksp.solve(rhs, solution) + if "cuda" in self.A.petscmat.type: + with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): + b_cu = PETSc.Vec() + b_cu.createCUDAWithArrays(rhs) + u = PETSc.Vec() + u.createCUDAWithArrays(solution) + self.ksp.solve(b_cu, u) + u.getArray() + + else: + + with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): + self.ksp.solve(rhs, solution) + # print(solution.view()) r = self.ksp.getConvergedReason() if r < 0: From 28ab1b5b968c4f09d7dae99df0f511e502600c00 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Wed, 21 Feb 2024 12:31:36 +0000 Subject: [PATCH 02/44] offload (not yet really) first try --- firedrake/linear_solver.py | 3 +- firedrake/preconditioners/offload.py | 147 +++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) create mode 100644 firedrake/preconditioners/offload.py diff --git a/firedrake/linear_solver.py b/firedrake/linear_solver.py index 4a618c8d38..555fea445c 100644 --- a/firedrake/linear_solver.py +++ b/firedrake/linear_solver.py @@ -188,13 +188,12 @@ def solve(self, x, b): u = PETSc.Vec() u.createCUDAWithArrays(solution) self.ksp.solve(b_cu, u) - u.getArray() + u.getArray() #how do this in one with solve before? else: with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): self.ksp.solve(rhs, solution) - # print(solution.view()) r = self.ksp.getConvergedReason() if r < 0: diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py new file mode 100644 index 0000000000..892645d141 --- /dev/null +++ b/firedrake/preconditioners/offload.py @@ -0,0 +1,147 @@ +#copied from assembled + +import abc + +from firedrake.preconditioners.base import PCBase +from firedrake.functionspace import FunctionSpace, MixedFunctionSpace +from firedrake.petsc import PETSc +from firedrake.ufl_expr import TestFunction, TrialFunction +import firedrake.dmhooks as dmhooks +from firedrake.dmhooks import get_function_space + +import petsc4py.PETSc # in firedrake.petsc? + +#outside: ksp.setOperators(A) + +__all__ = ("OffloadPC") + + +class OffloadPC(PCBase): #still PETSc PC object? + """Offload to GPU as PC to solve. + + Internally this makes a PETSc PC object that can be controlled by + options using the extra options prefix ``offload_``. + """ + + _prefix = "offload_" + + def initialize(self, pc): + A, P = pc.getOperators() #both needed?- forgot + + if pc.getType() != "python": + raise ValueError("Expecting PC type python") + opc = pc #opc? + appctx = self.get_appctx(pc) # + fcp = appctx.get("form_compiler_parameters") # + + #rest needed here? + V = get_function_space(pc.getDM()) + if len(V) == 1: + V = FunctionSpace(V.mesh(), V.ufl_element()) + else: + V = MixedFunctionSpace([V_ for V_ in V]) + test = TestFunction(V) + trial = TrialFunction(V) + + if P.type == "python": + context = P.getPythonContext() + # It only makes sense to preconditioner/invert a diagonal + # block in general. That's all we're going to allow. + if not context.on_diag: # + raise ValueError("Only makes sense to invert diagonal block") + + prefix = pc.getOptionsPrefix() + options_prefix = prefix + self._prefix + + mat_type = PETSc.Options().getString(options_prefix + "mat_type", "aij") #cuda + + (a, bcs) = self.form(pc, test, trial) + + self.P = allocate_matrix(a, bcs=bcs, + form_compiler_parameters=fcp, + mat_type=mat_type, + options_prefix=options_prefix) + self._assemble_P = TwoFormAssembler(a, tensor=self.P, bcs=bcs, + form_compiler_parameters=fcp).assemble + self._assemble_P() # + + # Transfer nullspace over + Pmat = self.P.petscmat + Pmat.setNullSpace(P.getNullSpace()) + tnullsp = P.getTransposeNullSpace() + if tnullsp.handle != 0: + Pmat.setTransposeNullSpace(tnullsp) + Pmat.setNearNullSpace(P.getNearNullSpace()) + + # Internally, we just set up a PC object that the user can configure + # however from the PETSc command line. Since PC allows the user to specify + # a KSP, we can do iterative by -assembled_pc_type ksp. + #? + +#same - matrix here + pc = PETSc.PC().create(comm=opc.comm) #comm? + pc.incrementTabLevel(1, parent=opc) # + + # We set a DM and an appropriate SNESContext on the constructed PC so one + # can do e.g. multigrid or patch solves. + dm = opc.getDM() + self._ctx_ref = self.new_snes_ctx(opc, a, bcs, mat_type, + fcp=fcp, options_prefix=options_prefix) + + pc.setDM(dm) #DM? + pc.setOptionsPrefix(options_prefix) + + #matrix to cuda + A_cu = petsc4py.PETSc.Mat() + A_cu.createDenseCUDA(A.petscmat.size) + A.petscmat.copy(A_cu) + A.petscmat = A_cu + + P_cu = petsc4py.PETSc.Mat() + P_cu.createDenseCUDA(A.petscmat.size) + Pmat.petscmat.copy(P_cu) + Pmat.petscmat = P_cu + + pc.setOperators(A_cu, P_cu) #right? + self.pc = pc + with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): + pc.setFromOptions() + + def update(self, pc): + self._assemble_P() + + def form(self, pc, test, trial): + _, P = pc.getOperators() + if P.getType() == "python": + context = P.getPythonContext() + return (context.a, context.row_bcs) + else: + context = dmhooks.get_appctx(pc.getDM()) + return (context.Jp or context.J, context._problem.bcs) + +#vectors here + def apply(self, pc, x, y): #y=b? + dm = pc.getDM() + with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref), : + b_cu = PETSc.Vec() #nonsense? + b_cu.createCUDAWithArrays(y) + u = PETSc.Vec() + u.createCUDAWithArrays(x) + self.pc.apply(x, y) + + def applyTranspose(self, pc, x, y): + dm = pc.getDM() + with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): + b_cu = PETSc.Vec() + b_cu.createCUDAWithArrays(y) + u = PETSc.Vec() + u.createCUDAWithArrays(x) + self.pc.applyTranspose(x, y) + + def view(self, pc, viewer=None): + super(AssembledPC, self).view(pc, viewer) + if hasattr(self, "pc"): + viewer.printfASCII("PC to apply inverse\n") + self.pc.view(viewer) + +#after pc where in lin solv? From 42d2e017356cb82056ad514eaf9b586eeb36fc89 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Sun, 25 Feb 2024 21:19:13 +0000 Subject: [PATCH 03/44] linear solver update cusparse --- firedrake/linear_solver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/firedrake/linear_solver.py b/firedrake/linear_solver.py index 555fea445c..06f4deadfa 100644 --- a/firedrake/linear_solver.py +++ b/firedrake/linear_solver.py @@ -181,14 +181,14 @@ def solve(self, x, b): else: acc = x.dat.vec_wo - if "cuda" in self.A.petscmat.type: + if "cu" in self.A.petscmat.type: #todo: cuda or cu? with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): b_cu = PETSc.Vec() b_cu.createCUDAWithArrays(rhs) u = PETSc.Vec() u.createCUDAWithArrays(solution) self.ksp.solve(b_cu, u) - u.getArray() #how do this in one with solve before? + u.getArray() else: From 8daaf5da0d9cdffed305c4d82c08129246eed30e Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Wed, 6 Mar 2024 14:34:02 +0000 Subject: [PATCH 04/44] some more changes --- firedrake/preconditioners/offload.py | 97 ++++++++++++---------------- 1 file changed, 42 insertions(+), 55 deletions(-) diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index 892645d141..f0e225381f 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -1,5 +1,3 @@ -#copied from assembled - import abc from firedrake.preconditioners.base import PCBase @@ -26,44 +24,32 @@ class OffloadPC(PCBase): #still PETSc PC object? _prefix = "offload_" def initialize(self, pc): - A, P = pc.getOperators() #both needed?- forgot + A, P = pc.getOperators() #P preconditioner - if pc.getType() != "python": - raise ValueError("Expecting PC type python") + if pc.getType() != "assembled": + raise ValueError("Expecting PC type assembled") #correct type? opc = pc #opc? - appctx = self.get_appctx(pc) # - fcp = appctx.get("form_compiler_parameters") # - - #rest needed here? - V = get_function_space(pc.getDM()) - if len(V) == 1: - V = FunctionSpace(V.mesh(), V.ufl_element()) - else: - V = MixedFunctionSpace([V_ for V_ in V]) - test = TestFunction(V) - trial = TrialFunction(V) - - if P.type == "python": + appctx = self.get_appctx(pc) + fcp = appctx.get("form_compiler_parameters") + + if P.type == "assembled": #not python value error - only assembled (preconditioner) context = P.getPythonContext() # It only makes sense to preconditioner/invert a diagonal # block in general. That's all we're going to allow. - if not context.on_diag: # + if not context.on_diag: #still? diagonal block? raise ValueError("Only makes sense to invert diagonal block") prefix = pc.getOptionsPrefix() options_prefix = prefix + self._prefix - mat_type = PETSc.Options().getString(options_prefix + "mat_type", "aij") #cuda + mat_type = PETSc.Options().getString(options_prefix + "mat_type", "aijcusparse") (a, bcs) = self.form(pc, test, trial) - self.P = allocate_matrix(a, bcs=bcs, + self.P = allocate_matrix(a, bcs=bcs, #eventually change allocate matrix form_compiler_parameters=fcp, mat_type=mat_type, options_prefix=options_prefix) - self._assemble_P = TwoFormAssembler(a, tensor=self.P, bcs=bcs, - form_compiler_parameters=fcp).assemble - self._assemble_P() # # Transfer nullspace over Pmat = self.P.petscmat @@ -79,7 +65,7 @@ def initialize(self, pc): #? #same - matrix here - pc = PETSc.PC().create(comm=opc.comm) #comm? + pc = PETSc.PC().create(comm=opc.comm) pc.incrementTabLevel(1, parent=opc) # # We set a DM and an appropriate SNESContext on the constructed PC so one @@ -88,7 +74,7 @@ def initialize(self, pc): self._ctx_ref = self.new_snes_ctx(opc, a, bcs, mat_type, fcp=fcp, options_prefix=options_prefix) - pc.setDM(dm) #DM? + pc.setDM(dm) pc.setOptionsPrefix(options_prefix) #matrix to cuda @@ -96,52 +82,53 @@ def initialize(self, pc): A_cu.createDenseCUDA(A.petscmat.size) A.petscmat.copy(A_cu) A.petscmat = A_cu + self._offload_A = A.petscmat #fishy P_cu = petsc4py.PETSc.Mat() P_cu.createDenseCUDA(A.petscmat.size) Pmat.petscmat.copy(P_cu) Pmat.petscmat = P_cu + self._offload_A = Pmat.petscmat #fishy - pc.setOperators(A_cu, P_cu) #right? + pc.setOperators(A_cu, P_cu) self.pc = pc with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): pc.setFromOptions() def update(self, pc): - self._assemble_P() - - def form(self, pc, test, trial): - _, P = pc.getOperators() - if P.getType() == "python": - context = P.getPythonContext() - return (context.a, context.row_bcs) - else: - context = dmhooks.get_appctx(pc.getDM()) - return (context.Jp or context.J, context._problem.bcs) - -#vectors here + self._offload_A() + + # def form(self, pc, test, trial): + # _, P = pc.getOperators() + # if P.getType() == "python": + # context = P.getPythonContext() + # return (context.a, context.row_bcs) + # else: + # context = dmhooks.get_appctx(pc.getDM()) + # return (context.Jp or context.J, context._problem.bcs) + +#vectors and solve def apply(self, pc, x, y): #y=b? dm = pc.getDM() with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref), : - b_cu = PETSc.Vec() #nonsense? + b_cu = PETSc.Vec() b_cu.createCUDAWithArrays(y) u = PETSc.Vec() u.createCUDAWithArrays(x) - self.pc.apply(x, y) - - def applyTranspose(self, pc, x, y): - dm = pc.getDM() - with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): - b_cu = PETSc.Vec() - b_cu.createCUDAWithArrays(y) - u = PETSc.Vec() - u.createCUDAWithArrays(x) - self.pc.applyTranspose(x, y) + self.pc.apply(x, y) #solve is here + u.getArray() #give vector back + + # def applyTranspose(self, pc, x, y): #same but other side + # dm = pc.getDM() + # with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): + # b_cu = PETSc.Vec() + # b_cu.createCUDAWithArrays(y) + # u = PETSc.Vec() + # u.createCUDAWithArrays(x) + # self.pc.applyTranspose(x, y) def view(self, pc, viewer=None): - super(AssembledPC, self).view(pc, viewer) + super().view(pc, viewer) if hasattr(self, "pc"): - viewer.printfASCII("PC to apply inverse\n") - self.pc.view(viewer) - -#after pc where in lin solv? + viewer.printfASCII("PC to solve on GPU\n") + self.pc.view(viewer) \ No newline at end of file From 0995127ed00d3dd27303f0f26f9be9b4e9a453a5 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Tue, 12 Mar 2024 13:03:03 +0000 Subject: [PATCH 05/44] cusparse convert - not done --- firedrake/linear_solver.py | 25 +++--- firedrake/preconditioners/offload.py | 122 +++++++++++---------------- 2 files changed, 62 insertions(+), 85 deletions(-) diff --git a/firedrake/linear_solver.py b/firedrake/linear_solver.py index 06f4deadfa..4eb210586f 100644 --- a/firedrake/linear_solver.py +++ b/firedrake/linear_solver.py @@ -57,6 +57,8 @@ def __init__(self, A, *, P=None, solver_parameters=None, solver_parameters = solving_utils.set_defaults(solver_parameters, A.arguments(), ksp_defaults=self.DEFAULT_KSP_PARAMETERS) + # todo: add offload to solver parameters - how? prefix? + self.A = A self.comm = A.comm self._comm = internal_comm(self.comm, self) @@ -181,19 +183,20 @@ def solve(self, x, b): else: acc = x.dat.vec_wo - if "cu" in self.A.petscmat.type: #todo: cuda or cu? - with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): - b_cu = PETSc.Vec() - b_cu.createCUDAWithArrays(rhs) - u = PETSc.Vec() - u.createCUDAWithArrays(solution) - self.ksp.solve(b_cu, u) - u.getArray() + # if "cu" in self.A.petscmat.type: # todo: cuda or cu? + # with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): + # b_cu = PETSc.Vec() + # b_cu.createCUDAWithArrays(rhs) + # u = PETSc.Vec() + # u.createCUDAWithArrays(solution) + # self.ksp.solve(b_cu, u) + # u.getArray() - else: + # else: + # instead: preconditioner - with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): - self.ksp.solve(rhs, solution) + with self.inserted_options(), b.dat.vec_ro as rhs, acc as solution, dmhooks.add_hooks(self.ksp.dm, self): + self.ksp.solve(rhs, solution) r = self.ksp.getConvergedReason() if r < 0: diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index f0e225381f..437c81deca 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -9,12 +9,13 @@ import petsc4py.PETSc # in firedrake.petsc? -#outside: ksp.setOperators(A) +# outside: ksp.setOperators(A) +# todo: for densecuda later- now only cusparse __all__ = ("OffloadPC") -class OffloadPC(PCBase): #still PETSc PC object? +class OffloadPC(PCBase): """Offload to GPU as PC to solve. Internally this makes a PETSc PC object that can be controlled by @@ -24,111 +25,84 @@ class OffloadPC(PCBase): #still PETSc PC object? _prefix = "offload_" def initialize(self, pc): - A, P = pc.getOperators() #P preconditioner + A, P = pc.getOperators() # P preconditioner if pc.getType() != "assembled": - raise ValueError("Expecting PC type assembled") #correct type? - opc = pc #opc? - appctx = self.get_appctx(pc) - fcp = appctx.get("form_compiler_parameters") - - if P.type == "assembled": #not python value error - only assembled (preconditioner) + raise ValueError("Expecting PC type assembled") # correct type? + outer_pc = pc + appctx = self.get_appctx(pc) + fcp = appctx.get("form_compiler_parameters") + + V = get_function_space(pc.getDM()) + if len(V) == 1: + V = FunctionSpace(V.mesh(), V.ufl_element()) + else: + V = MixedFunctionSpace([V_ for V_ in V]) + test = TestFunction(V) + trial = TrialFunction(V) + + (a, bcs) = self.form(pc, test, trial) + + if P.type == "assembled": # not python value error - only assembled (preconditioner) context = P.getPythonContext() # It only makes sense to preconditioner/invert a diagonal # block in general. That's all we're going to allow. - if not context.on_diag: #still? diagonal block? + if not context.on_diag: # still? diagonal block? raise ValueError("Only makes sense to invert diagonal block") - + prefix = pc.getOptionsPrefix() options_prefix = prefix + self._prefix - mat_type = PETSc.Options().getString(options_prefix + "mat_type", "aijcusparse") - - (a, bcs) = self.form(pc, test, trial) + mat_type = PETSc.Options().getString(options_prefix + "mat_type", "cusparse") # cuda? - self.P = allocate_matrix(a, bcs=bcs, #eventually change allocate matrix - form_compiler_parameters=fcp, - mat_type=mat_type, - options_prefix=options_prefix) + # eventually change allocate_matrix from assembled.py # Transfer nullspace over - Pmat = self.P.petscmat + Pmat = self.P.petscmat Pmat.setNullSpace(P.getNullSpace()) tnullsp = P.getTransposeNullSpace() if tnullsp.handle != 0: Pmat.setTransposeNullSpace(tnullsp) Pmat.setNearNullSpace(P.getNearNullSpace()) - # Internally, we just set up a PC object that the user can configure - # however from the PETSc command line. Since PC allows the user to specify - # a KSP, we can do iterative by -assembled_pc_type ksp. - #? + pc = PETSc.PC().create(comm=outer_pc.comm) + pc.incrementTabLevel(1, parent=outer_pc) -#same - matrix here - pc = PETSc.PC().create(comm=opc.comm) - pc.incrementTabLevel(1, parent=opc) # - - # We set a DM and an appropriate SNESContext on the constructed PC so one - # can do e.g. multigrid or patch solves. - dm = opc.getDM() - self._ctx_ref = self.new_snes_ctx(opc, a, bcs, mat_type, + # We set a DM and an appropriate SNESContext on the constructed PC + # so one can do e.g. multigrid or patch solves. + dm = outer_pc.getDM() + self._ctx_ref = self.new_snes_ctx(outer_pc, a, bcs, mat_type, fcp=fcp, options_prefix=options_prefix) - pc.setDM(dm) + pc.setDM(dm) pc.setOptionsPrefix(options_prefix) - #matrix to cuda - A_cu = petsc4py.PETSc.Mat() - A_cu.createDenseCUDA(A.petscmat.size) - A.petscmat.copy(A_cu) - A.petscmat = A_cu - self._offload_A = A.petscmat #fishy - - P_cu = petsc4py.PETSc.Mat() - P_cu.createDenseCUDA(A.petscmat.size) - Pmat.petscmat.copy(P_cu) - Pmat.petscmat = P_cu - self._offload_A = Pmat.petscmat #fishy + # matrix to cuda + P_cu = P.petscmat.convert(mat_type='aijcusparse') - pc.setOperators(A_cu, P_cu) + pc.setOperators(A, P_cu) self.pc = pc - with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): + with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): pc.setFromOptions() def update(self, pc): - self._offload_A() - - # def form(self, pc, test, trial): - # _, P = pc.getOperators() - # if P.getType() == "python": - # context = P.getPythonContext() - # return (context.a, context.row_bcs) - # else: - # context = dmhooks.get_appctx(pc.getDM()) - # return (context.Jp or context.J, context._problem.bcs) - -#vectors and solve - def apply(self, pc, x, y): #y=b? + _, P = pc.getOperators() + _, P_cu = self.pc.getOperators() + P.copy(P_cu) + +# vectors and solve + def apply(self, pc, x, y): # y=b? dm = pc.getDM() - with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref), : - b_cu = PETSc.Vec() - b_cu.createCUDAWithArrays(y) + with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): + b_cu = PETSc.Vec() + b_cu.createCUDAWithArrays(y) u = PETSc.Vec() u.createCUDAWithArrays(x) - self.pc.apply(x, y) #solve is here - u.getArray() #give vector back - - # def applyTranspose(self, pc, x, y): #same but other side - # dm = pc.getDM() - # with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): - # b_cu = PETSc.Vec() - # b_cu.createCUDAWithArrays(y) - # u = PETSc.Vec() - # u.createCUDAWithArrays(x) - # self.pc.applyTranspose(x, y) + self.pc.apply(x, y) # solve is here + u.getArray() # return vector def view(self, pc, viewer=None): super().view(pc, viewer) if hasattr(self, "pc"): viewer.printfASCII("PC to solve on GPU\n") - self.pc.view(viewer) \ No newline at end of file + self.pc.view(viewer) From 38fd7ba8cc1178f3e56d05ed4b838091bc0c8724 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Thu, 14 Mar 2024 17:38:16 +0000 Subject: [PATCH 06/44] last changes to offload --- firedrake/preconditioners/__init__.py | 1 + firedrake/preconditioners/offload.py | 37 +++++++++++++++++---------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/firedrake/preconditioners/__init__.py b/firedrake/preconditioners/__init__.py index 491a73657b..1cec69ab49 100644 --- a/firedrake/preconditioners/__init__.py +++ b/firedrake/preconditioners/__init__.py @@ -13,3 +13,4 @@ from firedrake.preconditioners.hiptmair import * # noqa: F401 from firedrake.preconditioners.facet_split import * # noqa: F401 from firedrake.preconditioners.bddc import * # noqa: F401 +from firedrake.preconditioners.offload import * # noqa: F401 diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index 437c81deca..9e5345b320 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -12,7 +12,7 @@ # outside: ksp.setOperators(A) # todo: for densecuda later- now only cusparse -__all__ = ("OffloadPC") +__all__ = ("OffloadPC",) class OffloadPC(PCBase): @@ -27,8 +27,6 @@ class OffloadPC(PCBase): def initialize(self, pc): A, P = pc.getOperators() # P preconditioner - if pc.getType() != "assembled": - raise ValueError("Expecting PC type assembled") # correct type? outer_pc = pc appctx = self.get_appctx(pc) fcp = appctx.get("form_compiler_parameters") @@ -55,15 +53,15 @@ def initialize(self, pc): mat_type = PETSc.Options().getString(options_prefix + "mat_type", "cusparse") # cuda? + # matrix to cuda + P_cu = P.convert(mat_type='aijcusparse') # eventually change allocate_matrix from assembled.py - # Transfer nullspace over - Pmat = self.P.petscmat - Pmat.setNullSpace(P.getNullSpace()) + P_cu.setNullSpace(P.getNullSpace()) tnullsp = P.getTransposeNullSpace() if tnullsp.handle != 0: - Pmat.setTransposeNullSpace(tnullsp) - Pmat.setNearNullSpace(P.getNearNullSpace()) + P_cu.setTransposeNullSpace(tnullsp) + P_cu.setNearNullSpace(P.getNearNullSpace()) pc = PETSc.PC().create(comm=outer_pc.comm) pc.incrementTabLevel(1, parent=outer_pc) @@ -71,15 +69,13 @@ def initialize(self, pc): # We set a DM and an appropriate SNESContext on the constructed PC # so one can do e.g. multigrid or patch solves. dm = outer_pc.getDM() - self._ctx_ref = self.new_snes_ctx(outer_pc, a, bcs, mat_type, - fcp=fcp, options_prefix=options_prefix) + self._ctx_ref = self.new_snes_ctx( + outer_pc, a, bcs, mat_type, + fcp=fcp, options_prefix=options_prefix + ) pc.setDM(dm) pc.setOptionsPrefix(options_prefix) - - # matrix to cuda - P_cu = P.petscmat.convert(mat_type='aijcusparse') - pc.setOperators(A, P_cu) self.pc = pc with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): @@ -90,6 +86,15 @@ def update(self, pc): _, P_cu = self.pc.getOperators() P.copy(P_cu) + def form(self, pc, test, trial): + _, P = pc.getOperators() + if P.getType() == "python": + context = P.getPythonContext() + return (context.a, context.row_bcs) + else: + context = dmhooks.get_appctx(pc.getDM()) + return (context.Jp or context.J, context._problem.bcs) + # vectors and solve def apply(self, pc, x, y): # y=b? dm = pc.getDM() @@ -101,8 +106,12 @@ def apply(self, pc, x, y): # y=b? self.pc.apply(x, y) # solve is here u.getArray() # return vector + def applyTranspose(self, pc, X, Y): + raise NotImplementedError + def view(self, pc, viewer=None): super().view(pc, viewer) + print("viewing PC") if hasattr(self, "pc"): viewer.printfASCII("PC to solve on GPU\n") self.pc.view(viewer) From fb9be7fc7172d5837abd8ccf44be07d407023139 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Tue, 23 Apr 2024 19:37:52 +0100 Subject: [PATCH 07/44] Commentary --- firedrake/preconditioners/offload.py | 36 ++++++++++++---------------- firedrake/solving.py | 25 ++++++++++--------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index 9e5345b320..c62515853d 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -1,5 +1,3 @@ -import abc - from firedrake.preconditioners.base import PCBase from firedrake.functionspace import FunctionSpace, MixedFunctionSpace from firedrake.petsc import PETSc @@ -7,16 +5,11 @@ import firedrake.dmhooks as dmhooks from firedrake.dmhooks import get_function_space -import petsc4py.PETSc # in firedrake.petsc? - -# outside: ksp.setOperators(A) -# todo: for densecuda later- now only cusparse - __all__ = ("OffloadPC",) class OffloadPC(PCBase): - """Offload to GPU as PC to solve. + """Offload PC from CPU to GPU and back. Internally this makes a PETSc PC object that can be controlled by options using the extra options prefix ``offload_``. @@ -41,28 +34,29 @@ def initialize(self, pc): (a, bcs) = self.form(pc, test, trial) - if P.type == "assembled": # not python value error - only assembled (preconditioner) + if P.type == "assembled": context = P.getPythonContext() # It only makes sense to preconditioner/invert a diagonal # block in general. That's all we're going to allow. - if not context.on_diag: # still? diagonal block? + if not context.on_diag: raise ValueError("Only makes sense to invert diagonal block") prefix = pc.getOptionsPrefix() options_prefix = prefix + self._prefix - mat_type = PETSc.Options().getString(options_prefix + "mat_type", "cusparse") # cuda? + mat_type = PETSc.Options().getString(options_prefix + "mat_type", "cusparse") - # matrix to cuda + # Convert matrix to ajicusparse P_cu = P.convert(mat_type='aijcusparse') - # eventually change allocate_matrix from assembled.py + # Transfer nullspace P_cu.setNullSpace(P.getNullSpace()) tnullsp = P.getTransposeNullSpace() if tnullsp.handle != 0: P_cu.setTransposeNullSpace(tnullsp) P_cu.setNearNullSpace(P.getNearNullSpace()) + # PC object set-up pc = PETSc.PC().create(comm=outer_pc.comm) pc.incrementTabLevel(1, parent=outer_pc) @@ -95,16 +89,16 @@ def form(self, pc, test, trial): context = dmhooks.get_appctx(pc.getDM()) return (context.Jp or context.J, context._problem.bcs) -# vectors and solve - def apply(self, pc, x, y): # y=b? + # Convert vectors to CUDA, solve and get solution on CPU back + def apply(self, pc, x, y): dm = pc.getDM() with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): - b_cu = PETSc.Vec() - b_cu.createCUDAWithArrays(y) - u = PETSc.Vec() - u.createCUDAWithArrays(x) - self.pc.apply(x, y) # solve is here - u.getArray() # return vector + y_cu = PETSc.Vec() + y_cu.createCUDAWithArrays(y) + x_cu = PETSc.Vec() + x_cu.createCUDAWithArrays(x) + self.pc.apply(x_cu, y_cu) + y.copy(y_cu) def applyTranspose(self, pc, X, Y): raise NotImplementedError diff --git a/firedrake/solving.py b/firedrake/solving.py index e71b13cc04..36d912b0bb 100644 --- a/firedrake/solving.py +++ b/firedrake/solving.py @@ -252,19 +252,18 @@ def _la_solve(A, x, b, **kwargs): options_prefix=options_prefix) if isinstance(x, firedrake.Vector): x = x.function - # linear MG doesn't need RHS, supply zero. - L = 0 - aP = None if P is None else P.a - lvp = vs.LinearVariationalProblem(A.a, L, x, bcs=A.bcs, aP=aP) - mat_type = A.mat_type - pmat_type = mat_type if P is None else P.mat_type - appctx = solver_parameters.get("appctx", {}) - ctx = solving_utils._SNESContext(lvp, - mat_type=mat_type, - pmat_type=pmat_type, - appctx=appctx, - options_prefix=options_prefix, - pre_apply_bcs=pre_apply_bcs) + if not isinstance(A, firedrake.matrix.AssembledMatrix): + # linear MG doesn't need RHS, supply zero. + lvp = vs.LinearVariationalProblem(a=A.a, L=0, u=x, bcs=A.bcs) + mat_type = A.mat_type + appctx = solver_parameters.get("appctx", {}) + ctx = solving_utils._SNESContext(lvp, + mat_type=mat_type, + pmat_type=mat_type, + appctx=appctx, + options_prefix=options_prefix) + else: + ctx = None dm = solver.ksp.dm with dmhooks.add_hooks(dm, solver, appctx=ctx): From b6c6d4c38fb2b114701e2e2d5b5c1050261ea8d6 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Thu, 9 May 2024 20:21:55 +0100 Subject: [PATCH 08/44] after meeting --- demos/helmholtz/helmholtz.txt | 57 +++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 demos/helmholtz/helmholtz.txt diff --git a/demos/helmholtz/helmholtz.txt b/demos/helmholtz/helmholtz.txt new file mode 100644 index 0000000000..95334ccb6a --- /dev/null +++ b/demos/helmholtz/helmholtz.txt @@ -0,0 +1,57 @@ +Main Stage 366614 +Main Stage;firedrake 44369 +Main Stage;firedrake;firedrake.solving.solve 86 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve 196 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve 140 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESFunctionEval 736 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESFunctionEval;ParLoopExecute 212 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESFunctionEval;ParLoopExecute;Parloop_Cells_wrap_form0_cell_integral 112 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESFunctionEval;ParLoopExecute;Parloop_Cells_wrap_form0_cell_integral;pyop2.global_kernel.GlobalKernel.compile 415552 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESFunctionEval;firedrake.tsfc_interface.compile_form 42597 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESJacobianEval 866 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESJacobianEval;ParLoopExecute 149 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESJacobianEval;ParLoopExecute;Parloop_Cells_wrap_form00_cell_integral 136 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.solve;SNESSolve;SNESJacobianEval;ParLoopExecute;Parloop_Cells_wrap_form00_cell_integral;pyop2.global_kernel.GlobalKernel.compile 407506 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.__init__ 1771 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.__init__;firedrake.tsfc_interface.compile_form 56423 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.__init__;firedrake.tsfc_interface.compile_form;firedrake.formmanipulation.split_form 1907 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.NonlinearVariationalSolver.__init__;firedrake.solving_utils._SNESContext.__init__ 618 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.LinearVariationalProblem.__init__ 145 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.LinearVariationalProblem.__init__;firedrake.ufl_expr.action 4387 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.LinearVariationalProblem.__init__;firedrake.variational_solver.NonlinearVariationalProblem.__init__ 332 +Main Stage;firedrake;firedrake.solving.solve;firedrake.variational_solver.LinearVariationalProblem.__init__;firedrake.variational_solver.NonlinearVariationalProblem.__init__;firedrake.ufl_expr.adjoint 2798 +Main Stage;firedrake;firedrake.function.Function.interpolate 342 +Main Stage;firedrake;firedrake.function.Function.interpolate;firedrake.assemble.assemble 5644 +Main Stage;firedrake;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate 29 +Main Stage;firedrake;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate;ParLoopExecute 298 +Main Stage;firedrake;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate;ParLoopExecute;Parloop_Cells_wrap_expression_kernel 204 +Main Stage;firedrake;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate;ParLoopExecute;Parloop_Cells_wrap_expression_kernel;pyop2.global_kernel.GlobalKernel.compile 682292 +Main Stage;firedrake;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.make_interpolator 40658 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write 2473 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write;firedrake.function.Function.interpolate 303 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write;firedrake.function.Function.interpolate;firedrake.assemble.assemble 1080 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate 23 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate;ParLoopExecute 328 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate;ParLoopExecute;Parloop_Cells_wrap_expression_kernel 165 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.SameMeshInterpolator._interpolate;ParLoopExecute;Parloop_Cells_wrap_expression_kernel;pyop2.global_kernel.GlobalKernel.compile 663410 +Main Stage;firedrake;firedrake.output.vtk_output.VTKFile.write;firedrake.function.Function.interpolate;firedrake.assemble.assemble;firedrake.interpolation.make_interpolator 55147 +Main Stage;firedrake;firedrake.__init__ 495196 +Main Stage;firedrake;firedrake.assemble.assemble 949 +Main Stage;firedrake;firedrake.assemble.assemble;ParLoopExecute 310 +Main Stage;firedrake;firedrake.assemble.assemble;ParLoopExecute;Parloop_Cells_wrap_form_cell_integral 95 +Main Stage;firedrake;firedrake.assemble.assemble;ParLoopExecute;Parloop_Cells_wrap_form_cell_integral;pyop2.global_kernel.GlobalKernel.compile 355507 +Main Stage;firedrake;firedrake.assemble.assemble;firedrake.tsfc_interface.compile_form 20219 +Main Stage;firedrake;CreateFunctionSpace 919 +Main Stage;firedrake;CreateFunctionSpace;CreateFunctionSpace 79 +Main Stage;firedrake;CreateFunctionSpace;CreateFunctionSpace;firedrake.functionspaceimpl.FunctionSpace.__init__ 165 +Main Stage;firedrake;CreateFunctionSpace;CreateFunctionSpace;firedrake.functionspaceimpl.FunctionSpace.__init__;firedrake.functionspacedata.get_shared_data 13 +Main Stage;firedrake;CreateFunctionSpace;CreateFunctionSpace;firedrake.functionspaceimpl.FunctionSpace.__init__;firedrake.functionspacedata.get_shared_data;firedrake.functionspacedata.FunctionSpaceData.__init__ 825 +Main Stage;firedrake;CreateFunctionSpace;CreateFunctionSpace;firedrake.functionspaceimpl.FunctionSpace.__init__;firedrake.functionspacedata.get_shared_data;firedrake.functionspacedata.FunctionSpaceData.__init__;FunctionSpaceData: CreateElement 1274 +Main Stage;firedrake;CreateFunctionSpace;CreateFunctionSpace;firedrake.functionspaceimpl.FunctionSpace.__init__;firedrake.functionspacedata.get_shared_data;firedrake.functionspacedata.FunctionSpaceData.__init__;firedrake.mesh.MeshTopology._facets 789 +Main Stage;firedrake;CreateFunctionSpace;CreateMesh 147 +Main Stage;firedrake;CreateFunctionSpace;CreateMesh;Mesh: numbering 376 +Main Stage;firedrake;firedrake.utility_meshes.UnitSquareMesh 12 +Main Stage;firedrake;firedrake.utility_meshes.UnitSquareMesh;firedrake.utility_meshes.SquareMesh 11 +Main Stage;firedrake;firedrake.utility_meshes.UnitSquareMesh;firedrake.utility_meshes.SquareMesh;firedrake.utility_meshes.RectangleMesh 834 +Main Stage;firedrake;firedrake.utility_meshes.UnitSquareMesh;firedrake.utility_meshes.SquareMesh;firedrake.utility_meshes.RectangleMesh;CreateMesh 676 +Main Stage;firedrake;firedrake.utility_meshes.UnitSquareMesh;firedrake.utility_meshes.SquareMesh;firedrake.utility_meshes.RectangleMesh;DMPlexInterp 382 From 69508abff35c3f10ac01696f580bb8b78a3d7090 Mon Sep 17 00:00:00 2001 From: Lilly Schweiger Date: Fri, 12 Jul 2024 15:46:59 +0100 Subject: [PATCH 09/44] Events --- firedrake/preconditioners/offload.py | 134 ++++++++++++++------------- 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index c62515853d..7a306ae24e 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -18,62 +18,64 @@ class OffloadPC(PCBase): _prefix = "offload_" def initialize(self, pc): - A, P = pc.getOperators() # P preconditioner - - outer_pc = pc - appctx = self.get_appctx(pc) - fcp = appctx.get("form_compiler_parameters") - - V = get_function_space(pc.getDM()) - if len(V) == 1: - V = FunctionSpace(V.mesh(), V.ufl_element()) - else: - V = MixedFunctionSpace([V_ for V_ in V]) - test = TestFunction(V) - trial = TrialFunction(V) - - (a, bcs) = self.form(pc, test, trial) - - if P.type == "assembled": - context = P.getPythonContext() - # It only makes sense to preconditioner/invert a diagonal - # block in general. That's all we're going to allow. - if not context.on_diag: - raise ValueError("Only makes sense to invert diagonal block") - - prefix = pc.getOptionsPrefix() - options_prefix = prefix + self._prefix - - mat_type = PETSc.Options().getString(options_prefix + "mat_type", "cusparse") - - # Convert matrix to ajicusparse - P_cu = P.convert(mat_type='aijcusparse') - - # Transfer nullspace - P_cu.setNullSpace(P.getNullSpace()) - tnullsp = P.getTransposeNullSpace() - if tnullsp.handle != 0: - P_cu.setTransposeNullSpace(tnullsp) - P_cu.setNearNullSpace(P.getNearNullSpace()) - - # PC object set-up - pc = PETSc.PC().create(comm=outer_pc.comm) - pc.incrementTabLevel(1, parent=outer_pc) - - # We set a DM and an appropriate SNESContext on the constructed PC - # so one can do e.g. multigrid or patch solves. - dm = outer_pc.getDM() - self._ctx_ref = self.new_snes_ctx( - outer_pc, a, bcs, mat_type, - fcp=fcp, options_prefix=options_prefix - ) - - pc.setDM(dm) - pc.setOptionsPrefix(options_prefix) - pc.setOperators(A, P_cu) - self.pc = pc - with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): - pc.setFromOptions() + with PETSc.Log.Event("Event: initialize offload"): # + A, P = pc.getOperators() + + outer_pc = pc + appctx = self.get_appctx(pc) + fcp = appctx.get("form_compiler_parameters") + + V = get_function_space(pc.getDM()) + if len(V) == 1: + V = FunctionSpace(V.mesh(), V.ufl_element()) + else: + V = MixedFunctionSpace([V_ for V_ in V]) + test = TestFunction(V) + trial = TrialFunction(V) + + (a, bcs) = self.form(pc, test, trial) + + if P.type == "assembled": + context = P.getPythonContext() + # It only makes sense to preconditioner/invert a diagonal + # block in general. That's all we're going to allow. + if not context.on_diag: + raise ValueError("Only makes sense to invert diagonal block") + + prefix = pc.getOptionsPrefix() + options_prefix = prefix + self._prefix + + mat_type = PETSc.Options().getString(options_prefix + "mat_type", "cusparse") + + # Convert matrix to ajicusparse + with PETSc.Log.Event("Event: matrix offload"): + P_cu = P.convert(mat_type='aijcusparse') # todo + + # Transfer nullspace + P_cu.setNullSpace(P.getNullSpace()) + tnullsp = P.getTransposeNullSpace() + if tnullsp.handle != 0: + P_cu.setTransposeNullSpace(tnullsp) + P_cu.setNearNullSpace(P.getNearNullSpace()) + + # PC object set-up + pc = PETSc.PC().create(comm=outer_pc.comm) + pc.incrementTabLevel(1, parent=outer_pc) + + # We set a DM and an appropriate SNESContext on the constructed PC + # so one can do e.g. multigrid or patch solves. + dm = outer_pc.getDM() + self._ctx_ref = self.new_snes_ctx( + outer_pc, a, bcs, mat_type, + fcp=fcp, options_prefix=options_prefix + ) + + pc.setDM(dm) + pc.setOptionsPrefix(options_prefix) + pc.setOperators(A, P_cu) + self.pc = pc + with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): + pc.setFromOptions() def update(self, pc): _, P = pc.getOperators() @@ -91,14 +93,18 @@ def form(self, pc, test, trial): # Convert vectors to CUDA, solve and get solution on CPU back def apply(self, pc, x, y): - dm = pc.getDM() - with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): - y_cu = PETSc.Vec() - y_cu.createCUDAWithArrays(y) - x_cu = PETSc.Vec() - x_cu.createCUDAWithArrays(x) - self.pc.apply(x_cu, y_cu) - y.copy(y_cu) + with PETSc.Log.Event("Event: apply offload"): # + dm = pc.getDM() + with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): + with PETSc.Log.Event("Event: vectors offload"): + y_cu = PETSc.Vec() # begin + y_cu.createCUDAWithArrays(y) + x_cu = PETSc.Vec() + x_cu.createCUDAWithArrays(x) # end + with PETSc.Log.Event("Event: solve"): + self.pc.apply(x_cu, y_cu) # + with PETSc.Log.Event("Event: vectors copy back"): + y.copy(y_cu) # def applyTranspose(self, pc, X, Y): raise NotImplementedError From c120d5bcc16dc723a0dd787e5e98036df56e4fb2 Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 18 Mar 2025 16:59:23 +0000 Subject: [PATCH 10/44] adding simple test for debugging --- simple_test.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 simple_test.py diff --git a/simple_test.py b/simple_test.py new file mode 100644 index 0000000000..d5852203a2 --- /dev/null +++ b/simple_test.py @@ -0,0 +1,45 @@ +from firedrake import * + +parameters = { + "ksp_type": "preonly", + "pc_type": "python", + "pc_python_type": "firedrake.OffloadPC", + "offload": { + "pc_type": "ksp", + "ksp": { + "ksp_type": "cg", + "ksp_view": None, + "ksp_rtol": "1e-10", + "ksp_monitor": None, + "pc_type": "sor", + } + } +} + +mesh = UnitSquareMesh(100, 100) +V = FunctionSpace(mesh, "CG", 1) +u = TrialFunction(V) +v = TestFunction(V) + +f = Function(V) +x, y = SpatialCoordinate(mesh) +f.interpolate(2*pi**2*sin(pi*x)*sin(pi*y)) + +# Equations +L = inner(grad(u), grad(v)) * dx +R = inner(v, f) * dx + +# Dirichlet boundary on all sides to 0 +bcs = DirichletBC(V, 0, "on_boundary") + +# Exact solution +sol = Function(V) +sol.interpolate(sin(pi*x)*sin(pi*y)) + +# Solution function +u_f = Function(V) + +problem = LinearVariationalProblem(L, R, u_f, bcs=bcs) +solver = LinearVariationalSolver(problem, solver_parameters=parameters) +solver.solve() +errornorm(problem.u, sol) From bbd958fdbfc3afa6d8a45a41f1ee9680156559eb Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 27 Mar 2025 13:19:28 +0000 Subject: [PATCH 11/44] duplicating to get around locking --- firedrake/preconditioners/offload.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index 7a306ae24e..eaee7ee18f 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -97,10 +97,14 @@ def apply(self, pc, x, y): dm = pc.getDM() with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): with PETSc.Log.Event("Event: vectors offload"): + # Ensure x is not locked by duplicating it + x_unlocked = x.duplicate() + x.copy(x_unlocked) + y_cu = PETSc.Vec() # begin y_cu.createCUDAWithArrays(y) x_cu = PETSc.Vec() - x_cu.createCUDAWithArrays(x) # end + x_cu.createCUDAWithArrays(x_unlocked) # end with PETSc.Log.Event("Event: solve"): self.pc.apply(x_cu, y_cu) # with PETSc.Log.Event("Event: vectors copy back"): From f6ed3626dc6735b9dc1327df095815924255f413 Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 27 Mar 2025 14:27:09 +0000 Subject: [PATCH 12/44] different fix for the lcoked vector --- firedrake/preconditioners/offload.py | 7 ++--- simple_test.py | 45 ---------------------------- 2 files changed, 2 insertions(+), 50 deletions(-) delete mode 100644 simple_test.py diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index eaee7ee18f..fdcaca375c 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -97,14 +97,11 @@ def apply(self, pc, x, y): dm = pc.getDM() with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref): with PETSc.Log.Event("Event: vectors offload"): - # Ensure x is not locked by duplicating it - x_unlocked = x.duplicate() - x.copy(x_unlocked) - y_cu = PETSc.Vec() # begin y_cu.createCUDAWithArrays(y) x_cu = PETSc.Vec() - x_cu.createCUDAWithArrays(x_unlocked) # end + # Passing a vec into another vec doesnt work because original is locked + x_cu.createCUDAWithArrays(x.array_r) with PETSc.Log.Event("Event: solve"): self.pc.apply(x_cu, y_cu) # with PETSc.Log.Event("Event: vectors copy back"): diff --git a/simple_test.py b/simple_test.py deleted file mode 100644 index d5852203a2..0000000000 --- a/simple_test.py +++ /dev/null @@ -1,45 +0,0 @@ -from firedrake import * - -parameters = { - "ksp_type": "preonly", - "pc_type": "python", - "pc_python_type": "firedrake.OffloadPC", - "offload": { - "pc_type": "ksp", - "ksp": { - "ksp_type": "cg", - "ksp_view": None, - "ksp_rtol": "1e-10", - "ksp_monitor": None, - "pc_type": "sor", - } - } -} - -mesh = UnitSquareMesh(100, 100) -V = FunctionSpace(mesh, "CG", 1) -u = TrialFunction(V) -v = TestFunction(V) - -f = Function(V) -x, y = SpatialCoordinate(mesh) -f.interpolate(2*pi**2*sin(pi*x)*sin(pi*y)) - -# Equations -L = inner(grad(u), grad(v)) * dx -R = inner(v, f) * dx - -# Dirichlet boundary on all sides to 0 -bcs = DirichletBC(V, 0, "on_boundary") - -# Exact solution -sol = Function(V) -sol.interpolate(sin(pi*x)*sin(pi*y)) - -# Solution function -u_f = Function(V) - -problem = LinearVariationalProblem(L, R, u_f, bcs=bcs) -solver = LinearVariationalSolver(problem, solver_parameters=parameters) -solver.solve() -errornorm(problem.u, sol) From a46028efd6fa6ba3d87998fc01206773a70d9d9f Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 27 Mar 2025 15:06:13 +0000 Subject: [PATCH 13/44] only install for now --- .github/workflows/build_cuda.yml | 90 ++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 .github/workflows/build_cuda.yml diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml new file mode 100644 index 0000000000..4d4bab5663 --- /dev/null +++ b/.github/workflows/build_cuda.yml @@ -0,0 +1,90 @@ +name: Install and test Firedrake (CUDA) + +on: + push: + branches: + - master + pull_request: + +concurrency: + # Cancels jobs running if new commits are pushed + group: > + ${{ github.workflow }}- + ${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Install and test Firedrake (Linux) + strategy: + # We want to know all of the tests which fail, so don't kill real if + # complex fails and vice-versa + fail-fast: false + matrix: + arch: [default] + runs-on: [self-hosted, Linux] + container: + image: firedrakeproject/firedrake-env:latest + env: + FIREDRAKE_CI: 1 + OMP_NUM_THREADS: 1 + steps: + - name: Pre-run cleanup + # Make sure the current directory is empty + run: find . -delete + + - uses: actions/checkout@v4 + with: + path: firedrake-repo + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get -y install \ + $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-system-packages) + + - name: Install PETSc + run: | + git clone --depth 1 https://github.com/firedrakeproject/petsc.git + cd petsc + # TODO update configure file + ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi + make + # TODO: This fails for some reason + # make check + + - name: Install Firedrake + id: install + run: | + # TODO update configure file for the exports + # export $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-env) + export PETSC_DIR=./petsc + export PETSC_ARCH=arch-firedrake-default + export MPI_HOME=$PETSC_DIR/$PETSC_ARCH + export CC=$PETSC_DIR/$PETSC_ARCH/bin/mpicc + export CXX=$PETSC_DIR/$PETSC_ARCH/bin/mpicxx + export MPICC=$CC + export MPI_HOME=$PETSC_DIR/$PETSC_ARCH + export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH + export HDF5_MPI=ON + python3 -m venv venv + . venv/bin/activate + : # Force a rebuild of petsc4py as the cached one will not link to the fresh + : # install of PETSc. A similar trick may be needed for compiled dependencies + : # like h5py or mpi4py if changing HDF5/MPI libraries. + pip cache remove petsc4py + pip install --verbose --no-binary h5py './firedrake-repo[ci]' + firedrake-clean + : # Extra test dependencies + pip install matplotlib vtk + pip list + + - name: Run smoke tests + run: | + . venv/bin/activate + firedrake-check + timeout-minutes: 10 + + - name: Post-run cleanup + if: always() + run: find . -delete From 57b7a9757943ce711b4ab5500912df777953c085 Mon Sep 17 00:00:00 2001 From: Olender Date: Mon, 31 Mar 2025 08:08:54 +0100 Subject: [PATCH 14/44] calling data to synchronize vector --- firedrake/preconditioners/offload.py | 1 + 1 file changed, 1 insertion(+) diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index fdcaca375c..4f8c57940a 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -104,6 +104,7 @@ def apply(self, pc, x, y): x_cu.createCUDAWithArrays(x.array_r) with PETSc.Log.Event("Event: solve"): self.pc.apply(x_cu, y_cu) # + tmp = y_cu.array_r # Calling data to synchronize vector with PETSc.Log.Event("Event: vectors copy back"): y.copy(y_cu) # From 74e4cfb0021e4830f8bffe8fb897479f3123544f Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 1 Apr 2025 14:08:33 +0100 Subject: [PATCH 15/44] adding first test --- firedrake/linear_solver.py | 2 +- firedrake/preconditioners/offload.py | 5 +- .../cuda/test_poisson_offloading_pc.py | 63 +++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 tests/firedrake/cuda/test_poisson_offloading_pc.py diff --git a/firedrake/linear_solver.py b/firedrake/linear_solver.py index 4eb210586f..90e077e05a 100644 --- a/firedrake/linear_solver.py +++ b/firedrake/linear_solver.py @@ -191,7 +191,7 @@ def solve(self, x, b): # u.createCUDAWithArrays(solution) # self.ksp.solve(b_cu, u) # u.getArray() - + # else: # instead: preconditioner diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index 4f8c57940a..88804ae057 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -103,8 +103,9 @@ def apply(self, pc, x, y): # Passing a vec into another vec doesnt work because original is locked x_cu.createCUDAWithArrays(x.array_r) with PETSc.Log.Event("Event: solve"): - self.pc.apply(x_cu, y_cu) # - tmp = y_cu.array_r # Calling data to synchronize vector + self.pc.apply(x_cu, y_cu) + # Calling data to synchronize vector + tmp = y_cu.array_r # noqa: F841 with PETSc.Log.Event("Event: vectors copy back"): y.copy(y_cu) # diff --git a/tests/firedrake/cuda/test_poisson_offloading_pc.py b/tests/firedrake/cuda/test_poisson_offloading_pc.py new file mode 100644 index 0000000000..97e0610fc1 --- /dev/null +++ b/tests/firedrake/cuda/test_poisson_offloading_pc.py @@ -0,0 +1,63 @@ +from firedrake import * +import numpy as np +import pytest + +# TODO: add marker for cuda pytests and something to check if cuda memory was really used +@pytest.mark.parametrize("ksp_type, pc_type", [("cg", "sor"), ("cg", "gamg"), ("preonly", "lu")]) +def test_poisson_on_cuda(ksp_type, pc_type): + + # Different tests for poisoon: cg and pctype sor, --ksp_type=cg --pc_type=gamg + print(f"Using ksp_type = {ksp_type}, and pc_type = {pc_type}.", flush=True) + + nested_parameters = { + "pc_type": "ksp", + "ksp": { + "ksp_type": ksp_type, + "ksp_view": None, + "ksp_rtol": "1e-10", + "ksp_monitor": None, + "pc_type": pc_type, + } + } + parameters = { + "ksp_type": "preonly", + "pc_type": "python", + "pc_python_type": "firedrake.OffloadPC", + "offload": nested_parameters, + } + + mesh = UnitSquareMesh(10, 10) + V = FunctionSpace(mesh, "CG", 1) + u = TrialFunction(V) + v = TestFunction(V) + + f = Function(V) + x, y = SpatialCoordinate(mesh) + f.interpolate(2*pi**2*sin(pi*x)*sin(pi*y)) + + # Equations + L = inner(grad(u), grad(v)) * dx + R = inner(v, f) * dx + + # Dirichlet boundary on all sides to 0 + bcs = DirichletBC(V, 0, "on_boundary") + + # Exact solution + sol = Function(V) + sol.interpolate(sin(pi*x)*sin(pi*y)) + + # Solution function + u_f = Function(V) + + problem = LinearVariationalProblem(L, R, u_f, bcs=bcs) + solver = LinearVariationalSolver(problem, solver_parameters=parameters) + solver.solve() + npsol = sol.dat.data[:] + npu_f = u_f.dat.data[:] + error = errornorm(problem.u, sol) + print(f"Error norm = {error}", flush=True) + assert error < 1.2e-2 + + +if __name__ == "__main__": + test_poisson_on_cuda() From 65bceb0f75ea692210e01e185f7ec9aacc96af88 Mon Sep 17 00:00:00 2001 From: Olender Date: Wed, 2 Apr 2025 10:11:55 +0100 Subject: [PATCH 16/44] adding kmv wave test --- tests/firedrake/cuda/test_wave_equation.py | 125 +++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 tests/firedrake/cuda/test_wave_equation.py diff --git a/tests/firedrake/cuda/test_wave_equation.py b/tests/firedrake/cuda/test_wave_equation.py new file mode 100644 index 0000000000..42eabe8cdd --- /dev/null +++ b/tests/firedrake/cuda/test_wave_equation.py @@ -0,0 +1,125 @@ +from firedrake import * +import os +import numpy as np +import finat +from firedrake.__future__ import interpolate +import matplotlib.pyplot as plt +import pytest + + +# TODO: add marker for cuda pytestss +def test_kmv_wave_propagation_cuda(): + nested_parameters = { + "ksp_type": "preonly", + "pc_type": "jacobi" + } + parameters = { + "ksp_type": "preonly", + "pc_type": "python", + "pc_python_type": "firedrake.OffloadPC", + "offload": nested_parameters, + } + + + # Choosing degree + degree = 4 + + # Setting up time variables + dt = 0.001 # time step in seconds + final_time = 0.5 # final time in seconds + total_steps = int(final_time / dt) + 1 + + # Setting up mesh parameters + nx, ny = 10, 10 + mesh = PeriodicRectangleMesh(nx, ny, 1.0, 1.0) + + # Acquisition geometry + frequency_peak = 5.0 # The dominant frequency of the Ricker wavelet in Hz. + offset = 0.2 + source_locations = [(0.5, 0.5)] + receiver_locations = [(0.5, 0.5 + offset)] + + # Setting up function space + V = FunctionSpace(mesh, "KMV", degree) + + # Velocity model + c = Constant(1.5) + + # Ricker wavelet definition + def ricker_wavelet(t, freq, amp=1.0, delay=0.2, delay_type="time"): + if delay_type == "multiples_of_minimun": + time_delay = delay * np.sqrt(6.0) / (np.pi * freq) + elif delay_type == "time": + time_delay = delay + t = t - time_delay + # t = t - delay / freq + tt = (np.pi * freq * t) ** 2 + return amp * (1.0 - (2.0) * tt) * np.exp((-1.0) * tt) + + # Using vertex only mesh + source_mesh = VertexOnlyMesh(mesh, source_locations) + V_s = FunctionSpace(source_mesh, "DG", 0) + d_s = Function(V_s) + d_s.interpolate(1.0) + source_cofunction = assemble(d_s * TestFunction(V_s) * dx) + q_s = Cofunction(V.dual()).interpolate(source_cofunction) + receiver_mesh = VertexOnlyMesh(mesh, receiver_locations) + V_r = FunctionSpace(receiver_mesh, "DG", 0) + f = Cofunction(V.dual()) + + true_data_receivers = [] + + # Setting up forward problem + u = TrialFunction(V) + v = TestFunction(V) + u_np1 = Function(V) # timestep n+1 + u_n = Function(V) # timestep n + u_nm1 = Function(V) # timestep n-1 + # Quadrature rule for lumped mass matrix. + quad_rule = finat.quadrature.make_quadrature(V.finat_element.cell, V.ufl_element().degree(), "KMV") + m = (1 / (c * c)) + time_term = m * ((u - 2.0 * u_n + u_nm1) / Constant(dt**2)) * v * dx(scheme=quad_rule) + nf = (1 / c) * ((u_n - u_nm1) / dt) * v * ds + a = dot(grad(u_n), grad(v)) * dx(scheme=quad_rule) + F = time_term + a + nf + lin_var = LinearVariationalProblem(lhs(F), rhs(F) + f, u_np1) + # Since the linear system matrix is diagonal, the solver parameters are set to construct a solver, + # which applies a single step of Jacobi preconditioning. + + solver = LinearVariationalSolver(lin_var,solver_parameters=parameters) + + interpolate_receivers = interpolate(u_np1, V_r) + + # Looping in time + for step in range(total_steps): + if step % 100 == 0: + print(f"For time = {step*dt}s") + f.assign(ricker_wavelet(step * dt, frequency_peak) * q_s) + solver.solve() + u_nm1.assign(u_n) + u_n.assign(u_np1) + rec_out = assemble(interpolate_receivers) + true_data_receivers.append(rec_out.dat.data[:]) + + rec_matrix = np.matrix(true_data_receivers) + + # Hard coded values from an analytical solution + min_value = -0.0571 + max_value = 0.0946 + min_location = 0.269 + max_location = 0.352 + + correct_min_loc = np.isclose(min_location, np.argmin(rec_matrix)*dt) + correct_min = np.isclose(min_value, np.min(rec_matrix), rtol=1e-2) + correct_max = np.isclose(max_value, np.max(rec_matrix), rtol=1e-2) + correct_max_loc = np.isclose(max_location, np.argmax(rec_matrix)*dt) + + print(f"Correct minimum and its location: {correct_min} and {correct_min_loc}.") + print(f"Correct maximum and its location: {correct_max} and {correct_max_loc}.") + + print("END", flush=True) + assert all([correct_min_loc, correct_min, correct_max, correct_max_loc]) + + +if __name__ == "__main__": + test_kmv_wave_propagation_cuda() From c20a3e76e3ec32de58bb18bd416d0b3a00538a3d Mon Sep 17 00:00:00 2001 From: Olender Date: Wed, 2 Apr 2025 10:56:45 +0100 Subject: [PATCH 17/44] minor fix --- tests/firedrake/cuda/test_wave_equation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/firedrake/cuda/test_wave_equation.py b/tests/firedrake/cuda/test_wave_equation.py index 42eabe8cdd..2f3da10c17 100644 --- a/tests/firedrake/cuda/test_wave_equation.py +++ b/tests/firedrake/cuda/test_wave_equation.py @@ -31,7 +31,7 @@ def test_kmv_wave_propagation_cuda(): # Setting up mesh parameters nx, ny = 10, 10 - mesh = PeriodicRectangleMesh(nx, ny, 1.0, 1.0) + mesh = RectangleMesh(nx, ny, 1.0, 1.0) # Acquisition geometry frequency_peak = 5.0 # The dominant frequency of the Ricker wavelet in Hz. From 25ea7182b22a9ab961e25b07a05e2da79a109147 Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 3 Apr 2025 20:01:43 +0100 Subject: [PATCH 18/44] minor changes --- tests/firedrake/cuda/test_wave_equation.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/firedrake/cuda/test_wave_equation.py b/tests/firedrake/cuda/test_wave_equation.py index 2f3da10c17..38e9dfb5d7 100644 --- a/tests/firedrake/cuda/test_wave_equation.py +++ b/tests/firedrake/cuda/test_wave_equation.py @@ -3,7 +3,6 @@ import numpy as np import finat from firedrake.__future__ import interpolate -import matplotlib.pyplot as plt import pytest @@ -104,15 +103,16 @@ def ricker_wavelet(t, freq, amp=1.0, delay=0.2, delay_type="time"): rec_matrix = np.matrix(true_data_receivers) # Hard coded values from an analytical solution - min_value = -0.0571 - max_value = 0.0946 - min_location = 0.269 - max_location = 0.352 + # Hard coded values from an analytical solution + min_value = -0.05708 + max_value = 0.09467 + min_location = 0.2701 + max_location = 0.3528 - correct_min_loc = np.isclose(min_location, np.argmin(rec_matrix)*dt) + correct_min_loc = np.isclose(min_location, np.argmin(rec_matrix)*dt, rtol=1e-2) correct_min = np.isclose(min_value, np.min(rec_matrix), rtol=1e-2) correct_max = np.isclose(max_value, np.max(rec_matrix), rtol=1e-2) - correct_max_loc = np.isclose(max_location, np.argmax(rec_matrix)*dt) + correct_max_loc = np.isclose(max_location, np.argmax(rec_matrix)*dt, rtol=1e-2) print(f"Correct minimum and its location: {correct_min} and {correct_min_loc}.") print(f"Correct maximum and its location: {correct_max} and {correct_max_loc}.") From 3350bde897b7787465ae3f871e6e70417f1e4469 Mon Sep 17 00:00:00 2001 From: Olender Date: Fri, 4 Apr 2025 12:19:32 +0100 Subject: [PATCH 19/44] offload now subclass of assembledpc --- firedrake/linear_solver.py | 1 - firedrake/preconditioners/offload.py | 67 ++++------------------------ 2 files changed, 9 insertions(+), 59 deletions(-) diff --git a/firedrake/linear_solver.py b/firedrake/linear_solver.py index ca801e22f4..1e060ce906 100644 --- a/firedrake/linear_solver.py +++ b/firedrake/linear_solver.py @@ -87,4 +87,3 @@ def solve(self, x, b): self.b.assign(b) super().solve() x.assign(self.x) - diff --git a/firedrake/preconditioners/offload.py b/firedrake/preconditioners/offload.py index 88804ae057..451ce97c6a 100644 --- a/firedrake/preconditioners/offload.py +++ b/firedrake/preconditioners/offload.py @@ -1,14 +1,11 @@ -from firedrake.preconditioners.base import PCBase -from firedrake.functionspace import FunctionSpace, MixedFunctionSpace +from firedrake.preconditioners.assembled import AssembledPC from firedrake.petsc import PETSc -from firedrake.ufl_expr import TestFunction, TrialFunction import firedrake.dmhooks as dmhooks -from firedrake.dmhooks import get_function_space __all__ = ("OffloadPC",) -class OffloadPC(PCBase): +class OffloadPC(AssembledPC): """Offload PC from CPU to GPU and back. Internally this makes a PETSc PC object that can be controlled by @@ -18,69 +15,23 @@ class OffloadPC(PCBase): _prefix = "offload_" def initialize(self, pc): - with PETSc.Log.Event("Event: initialize offload"): # - A, P = pc.getOperators() - - outer_pc = pc - appctx = self.get_appctx(pc) - fcp = appctx.get("form_compiler_parameters") - - V = get_function_space(pc.getDM()) - if len(V) == 1: - V = FunctionSpace(V.mesh(), V.ufl_element()) - else: - V = MixedFunctionSpace([V_ for V_ in V]) - test = TestFunction(V) - trial = TrialFunction(V) - - (a, bcs) = self.form(pc, test, trial) - - if P.type == "assembled": - context = P.getPythonContext() - # It only makes sense to preconditioner/invert a diagonal - # block in general. That's all we're going to allow. - if not context.on_diag: - raise ValueError("Only makes sense to invert diagonal block") - - prefix = pc.getOptionsPrefix() - options_prefix = prefix + self._prefix + super().initialize(pc) - mat_type = PETSc.Options().getString(options_prefix + "mat_type", "cusparse") + with PETSc.Log.Event("Event: initialize offload"): + A, P = pc.getOperators() # Convert matrix to ajicusparse + mat_type = PETSc.Options().getString(self._prefix + "mat_type", "cusparse") with PETSc.Log.Event("Event: matrix offload"): P_cu = P.convert(mat_type='aijcusparse') # todo # Transfer nullspace P_cu.setNullSpace(P.getNullSpace()) - tnullsp = P.getTransposeNullSpace() - if tnullsp.handle != 0: - P_cu.setTransposeNullSpace(tnullsp) + P_cu.setTransposeNullSpace(P.getTransposeNullSpace()) P_cu.setNearNullSpace(P.getNearNullSpace()) - # PC object set-up - pc = PETSc.PC().create(comm=outer_pc.comm) - pc.incrementTabLevel(1, parent=outer_pc) - - # We set a DM and an appropriate SNESContext on the constructed PC - # so one can do e.g. multigrid or patch solves. - dm = outer_pc.getDM() - self._ctx_ref = self.new_snes_ctx( - outer_pc, a, bcs, mat_type, - fcp=fcp, options_prefix=options_prefix - ) - - pc.setDM(dm) - pc.setOptionsPrefix(options_prefix) - pc.setOperators(A, P_cu) - self.pc = pc - with dmhooks.add_hooks(dm, self, appctx=self._ctx_ref, save=False): - pc.setFromOptions() - - def update(self, pc): - _, P = pc.getOperators() - _, P_cu = self.pc.getOperators() - P.copy(P_cu) + # Update preconditioner with GPU matrix + self.pc.setOperators(A, P_cu) def form(self, pc, test, trial): _, P = pc.getOperators() From 763fe6b270ee2576981b1bb999a9e2d16d63b527 Mon Sep 17 00:00:00 2001 From: Olender Date: Fri, 4 Apr 2025 12:38:25 +0100 Subject: [PATCH 20/44] adding tests in CI --- .github/workflows/build_cuda.yml | 25 ++++++++++++++----- tests/firedrake/conftest.py | 16 +++++++++++- .../cuda/test_poisson_offloading_pc.py | 1 + tests/firedrake/cuda/test_wave_equation.py | 2 +- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 4d4bab5663..895a416a6f 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -1,3 +1,5 @@ +#TODO: Fix and then move this as matrix arch into build.yml and + name: Install and test Firedrake (CUDA) on: @@ -24,11 +26,18 @@ jobs: arch: [default] runs-on: [self-hosted, Linux] container: - image: firedrakeproject/firedrake-env:latest + image: nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04 + options: --gpus all env: FIREDRAKE_CI: 1 OMP_NUM_THREADS: 1 steps: + - name: Fix HOME + # For unknown reasons GitHub actions overwrite HOME to /github/home + # which will break everything unless fixed + # (https://github.com/actions/runner/issues/863) + run: echo "HOME=/root" >> "$GITHUB_ENV" + - name: Pre-run cleanup # Make sure the current directory is empty run: find . -delete @@ -39,8 +48,12 @@ jobs: - name: Install system dependencies run: | - sudo apt-get update - sudo apt-get -y install \ + apt-get update + apt-get install -y curl + apt-get install -y git + apt-get install -y python3 + apt install -y python3.12-venv + apt-get -y install \ $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-system-packages) - name: Install PETSc @@ -49,16 +62,16 @@ jobs: cd petsc # TODO update configure file ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi - make + make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all # TODO: This fails for some reason - # make check + make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" - name: Install Firedrake id: install run: | # TODO update configure file for the exports # export $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-env) - export PETSC_DIR=./petsc + export PETSC_DIR=/__w/firedrake/firedrake/petsc export PETSC_ARCH=arch-firedrake-default export MPI_HOME=$PETSC_DIR/$PETSC_ARCH export CC=$PETSC_DIR/$PETSC_ARCH/bin/mpicc diff --git a/tests/firedrake/conftest.py b/tests/firedrake/conftest.py index 10059578c5..b056d96b94 100644 --- a/tests/firedrake/conftest.py +++ b/tests/firedrake/conftest.py @@ -7,7 +7,7 @@ os.environ["FIREDRAKE_DISABLE_OPTIONS_LEFT"] = "1" import pytest -from firedrake.petsc import PETSc, get_external_packages +from firedrake.petsc import PETSc, get_external_packages, get_petsc_variables def _skip_test_dependency(dependency): @@ -145,11 +145,21 @@ def pytest_configure(config): "markers", "skipnetgen: mark as skipped if netgen and ngsPETSc is not installed" ) + config.addinivalue_line( + "markers", + "skipcuda: mark as skipped if CUDA is not available" + ) def pytest_collection_modifyitems(session, config, items): from firedrake.utils import complex_mode, SLATE_SUPPORTS_COMPLEX + try: + get_petsc_variables()["CUDA_VERSION"] + cuda_unavailable = False + except: + cuda_unavailable = True + for item in items: if complex_mode: if item.get_closest_marker("skipcomplex") is not None: @@ -160,6 +170,10 @@ def pytest_collection_modifyitems(session, config, items): if item.get_closest_marker("skipreal") is not None: item.add_marker(pytest.mark.skip(reason="Test makes no sense unless in complex mode")) + if cuda_unavailable: + if item.get_closest_marker("skipcuda") is not None: + item.add_marker(pytest.mark.skip(reason="CUDA not available")) + for dep, marker, reason in dependency_skip_markers_and_reasons: if _skip_test_dependency(dep) and item.get_closest_marker(marker) is not None: item.add_marker(pytest.mark.skip(reason)) diff --git a/tests/firedrake/cuda/test_poisson_offloading_pc.py b/tests/firedrake/cuda/test_poisson_offloading_pc.py index 97e0610fc1..3ad749e686 100644 --- a/tests/firedrake/cuda/test_poisson_offloading_pc.py +++ b/tests/firedrake/cuda/test_poisson_offloading_pc.py @@ -3,6 +3,7 @@ import pytest # TODO: add marker for cuda pytests and something to check if cuda memory was really used +@pytest.mark.skipcuda @pytest.mark.parametrize("ksp_type, pc_type", [("cg", "sor"), ("cg", "gamg"), ("preonly", "lu")]) def test_poisson_on_cuda(ksp_type, pc_type): diff --git a/tests/firedrake/cuda/test_wave_equation.py b/tests/firedrake/cuda/test_wave_equation.py index 38e9dfb5d7..d24c2741bf 100644 --- a/tests/firedrake/cuda/test_wave_equation.py +++ b/tests/firedrake/cuda/test_wave_equation.py @@ -6,7 +6,7 @@ import pytest -# TODO: add marker for cuda pytestss +@pytest.mark.skipcuda def test_kmv_wave_propagation_cuda(): nested_parameters = { "ksp_type": "preonly", From fc13aa273b0b19225aad380006bb875ab9d48aef Mon Sep 17 00:00:00 2001 From: Olender Date: Wed, 9 Apr 2025 14:56:31 +0100 Subject: [PATCH 21/44] checking if run tests gets the tests with cuda marker --- .github/workflows/build_cuda.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 895a416a6f..17e5f7238c 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -97,6 +97,16 @@ jobs: . venv/bin/activate firedrake-check timeout-minutes: 10 + + - name: Run tests (nprocs = 1) + # Run even if earlier tests failed + if: ${{ success() || steps.install.conclusion == 'success' }} + run: | + . venv/bin/activate + : # Use pytest-xdist here so we can have a single collated output (not possible + : # for parallel tests) + firedrake-run-split-tests 1 1 "-n 12 $EXTRA_PYTEST_ARGS" + timeout-minutes: 60 - name: Post-run cleanup if: always() From af7e6ce17efab0496c9a44d623a7bd4c027a5686 Mon Sep 17 00:00:00 2001 From: Alexandre Olender <45005909+Olender@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:03:38 +0100 Subject: [PATCH 22/44] Update .github/workflows/build_cuda.yml Co-authored-by: Connor Ward --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 17e5f7238c..af7bc536ba 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -24,7 +24,7 @@ jobs: fail-fast: false matrix: arch: [default] - runs-on: [self-hosted, Linux] + runs-on: [self-hosted, Linux, gpu] container: image: nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04 options: --gpus all From c988926413c43818e91977b663d66525a7abc047 Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 10 Apr 2025 16:06:56 +0100 Subject: [PATCH 23/44] adding env options --- .github/workflows/build_cuda.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index af7bc536ba..227a706085 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -29,8 +29,14 @@ jobs: image: nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04 options: --gpus all env: - FIREDRAKE_CI: 1 + OMPI_ALLOW_RUN_AS_ROOT: 1 + OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 OMP_NUM_THREADS: 1 + OPENBLAS_NUM_THREADS: 1 + FIREDRAKE_CI: 1 + PYOP2_CI_TESTS: 1 + PYOP2_SPMD_STRICT: 1 + EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 firedrake-repo/tests/firedrake steps: - name: Fix HOME # For unknown reasons GitHub actions overwrite HOME to /github/home From d075453ed3f0f88fee742df225219e05380c51de Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 10 Apr 2025 17:00:04 +0100 Subject: [PATCH 24/44] trying to figure out whats wrong with petsc4py now --- .github/workflows/build_cuda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 227a706085..1dbad35f92 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -92,6 +92,7 @@ jobs: : # install of PETSc. A similar trick may be needed for compiled dependencies : # like h5py or mpi4py if changing HDF5/MPI libraries. pip cache remove petsc4py + python -c "import petsc4py; print(petsc4py.get_config())" pip install --verbose --no-binary h5py './firedrake-repo[ci]' firedrake-clean : # Extra test dependencies From af69f87eeac13c47806fa7d40eafe3fdbea015de Mon Sep 17 00:00:00 2001 From: Olender Date: Mon, 14 Apr 2025 15:25:03 +0100 Subject: [PATCH 25/44] wip --- .github/workflows/build_cuda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 1dbad35f92..533b5d254b 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -92,7 +92,8 @@ jobs: : # install of PETSc. A similar trick may be needed for compiled dependencies : # like h5py or mpi4py if changing HDF5/MPI libraries. pip cache remove petsc4py - python -c "import petsc4py; print(petsc4py.get_config())" + pip cache remove slepc4py + # python -c "import petsc4py; print(petsc4py.get_config())" pip install --verbose --no-binary h5py './firedrake-repo[ci]' firedrake-clean : # Extra test dependencies From 84f8851ac8b6972129f40be9dcddf67d5a5712c6 Mon Sep 17 00:00:00 2001 From: Olender Date: Mon, 14 Apr 2025 16:00:08 +0100 Subject: [PATCH 26/44] updating PETSC --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 533b5d254b..0da79b889f 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -64,7 +64,7 @@ jobs: - name: Install PETSc run: | - git clone --depth 1 https://github.com/firedrakeproject/petsc.git + git clone --depth 1 --branch $(python3 firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git cd petsc # TODO update configure file ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi From b00c615f063922c0dd01556f54a0464620e15ead Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 07:07:58 +0100 Subject: [PATCH 27/44] wip --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 0da79b889f..0c5dad0012 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -64,7 +64,7 @@ jobs: - name: Install PETSc run: | - git clone --depth 1 --branch $(python3 firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git + git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git cd petsc # TODO update configure file ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi From 9e9fe080512ec4d623e6d47df69a5a89ca2f889f Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 07:52:41 +0100 Subject: [PATCH 28/44] wip --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 0c5dad0012..c1a531691e 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -113,7 +113,7 @@ jobs: . venv/bin/activate : # Use pytest-xdist here so we can have a single collated output (not possible : # for parallel tests) - firedrake-run-split-tests 1 1 "-n 12 $EXTRA_PYTEST_ARGS" + firedrake-run-split-tests 1 1 "-n 12 $EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake timeout-minutes: 60 - name: Post-run cleanup From 7a3c5da821529be7905aa7cadd749ec739caa67c Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 09:51:29 +0100 Subject: [PATCH 29/44] wip --- .github/workflows/build_cuda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index c1a531691e..09dd5a1c9e 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -59,6 +59,7 @@ jobs: apt-get install -y git apt-get install -y python3 apt install -y python3.12-venv + apt-get install -y parallel apt-get -y install \ $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-system-packages) From 7f69823a9137c85153e81010f9b1e57818d0a3a1 Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 10:49:16 +0100 Subject: [PATCH 30/44] adding slepc --- .github/workflows/build_cuda.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 09dd5a1c9e..04e091c456 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -68,7 +68,7 @@ jobs: git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git cd petsc # TODO update configure file - ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi + ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi --download-slepc make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all # TODO: This fails for some reason make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" @@ -86,6 +86,7 @@ jobs: export MPICC=$CC export MPI_HOME=$PETSC_DIR/$PETSC_ARCH export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH + export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_MPI=ON python3 -m venv venv . venv/bin/activate From c23f37ee74426ecd6809b64d0365986b7ca374b4 Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 11:50:50 +0100 Subject: [PATCH 31/44] wip --- .github/workflows/build_cuda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 04e091c456..e1ea6a1677 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -95,6 +95,7 @@ jobs: : # like h5py or mpi4py if changing HDF5/MPI libraries. pip cache remove petsc4py pip cache remove slepc4py + pip cache remove h5py # python -c "import petsc4py; print(petsc4py.get_config())" pip install --verbose --no-binary h5py './firedrake-repo[ci]' firedrake-clean From 58db56bb19a95e1670df83d7c9863f33f6e88b51 Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 14:47:58 +0100 Subject: [PATCH 32/44] wip --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index e1ea6a1677..274a3e43c5 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -116,7 +116,7 @@ jobs: . venv/bin/activate : # Use pytest-xdist here so we can have a single collated output (not possible : # for parallel tests) - firedrake-run-split-tests 1 1 "-n 12 $EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake + firedrake-run-split-tests 1 1 "-n 8 $EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake timeout-minutes: 60 - name: Post-run cleanup From a380acfdfc18b0744394e6a38f08c564da4d10f6 Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 15:52:38 +0100 Subject: [PATCH 33/44] wip --- .github/workflows/build_cuda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 274a3e43c5..465f039399 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -87,6 +87,7 @@ jobs: export MPI_HOME=$PETSC_DIR/$PETSC_ARCH export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH + export HDF5_DIR=$PETSC_DIR/$HDF5_DIR export HDF5_MPI=ON python3 -m venv venv . venv/bin/activate From 18f4daa1199872a4741d31ba705bd9713e7f4728 Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 15 Apr 2025 16:39:04 +0100 Subject: [PATCH 34/44] wip --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 465f039399..2f17496c1f 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -87,7 +87,7 @@ jobs: export MPI_HOME=$PETSC_DIR/$PETSC_ARCH export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH - export HDF5_DIR=$PETSC_DIR/$HDF5_DIR + export HDF5_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_MPI=ON python3 -m venv venv . venv/bin/activate From 319aa19bb123ab0db967ed14de17f0941cd11f01 Mon Sep 17 00:00:00 2001 From: Olender Date: Wed, 16 Apr 2025 15:01:42 +0100 Subject: [PATCH 35/44] wip --- .github/workflows/build_cuda.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 2f17496c1f..34ff77a82d 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -88,6 +88,7 @@ jobs: export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_DIR=$PETSC_DIR/$PETSC_ARCH + export HYPRE_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_MPI=ON python3 -m venv venv . venv/bin/activate From bbf1825fa588da1e542eaf309130d5b47a5a0a2f Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 17 Apr 2025 14:12:19 +0100 Subject: [PATCH 36/44] wip --- .github/workflows/build_cuda.yml | 2 +- mfe.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 mfe.py diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 34ff77a82d..e109185db0 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -68,7 +68,7 @@ jobs: git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git cd petsc # TODO update configure file - ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi --download-slepc + ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-mpich --download-slepc make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all # TODO: This fails for some reason make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" diff --git a/mfe.py b/mfe.py new file mode 100644 index 0000000000..2fac75246e --- /dev/null +++ b/mfe.py @@ -0,0 +1,17 @@ +from firedrake import * + + +# Setting up mesh parameters +nx, ny = 20, 20 +mesh = RectangleMesh(nx, ny, 1.0, 1.0) + + +# Setting up function space +degree = 4 +V = FunctionSpace(mesh, "CG", degree) + +# Using vertex only mesh +source_locations = [(0.5, 0.5)] +source_mesh = VertexOnlyMesh(mesh, source_locations) + +print("END", flush=True) From 07a2a202c00448a8df35ea4645663f61ab51317b Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 22 Apr 2025 15:17:56 +0100 Subject: [PATCH 37/44] wip --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index e109185db0..f962e1d2e6 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -68,7 +68,7 @@ jobs: git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git cd petsc # TODO update configure file - ./configure --with-make-np=12 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-mpich --download-slepc + ./configure --with-make-np=8 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-mpich --download-slepc make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all # TODO: This fails for some reason make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" From 97e41fa1d04a7b15f9562fc3efed3ba8313afe6b Mon Sep 17 00:00:00 2001 From: Olender Date: Wed, 23 Apr 2025 09:51:09 +0100 Subject: [PATCH 38/44] wip --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index f962e1d2e6..4fd79dfd64 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -71,7 +71,7 @@ jobs: ./configure --with-make-np=8 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-mpich --download-slepc make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all # TODO: This fails for some reason - make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" + # make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" - name: Install Firedrake id: install From 25d356bdeb06c617f9ec46b6d824c471ecf05d20 Mon Sep 17 00:00:00 2001 From: Olender Date: Wed, 23 Apr 2025 13:46:26 +0100 Subject: [PATCH 39/44] back to openmpi --- .github/workflows/build_cuda.yml | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 4fd79dfd64..e91cbf206c 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -60,18 +60,33 @@ jobs: apt-get install -y python3 apt install -y python3.12-venv apt-get install -y parallel - apt-get -y install \ - $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-system-packages) + apt-get install -y bison + apt-get install -y cmake + apt-get install -y flex + apt-get install -y gfortran + apt-get install -y libopenblas-dev + apt-get install -y ninja-build + apt-get install -y pkg-config + apt-get install -y python3-dev + apt-get install -y libfftw3-dev + apt-get install -y libhwloc-dev + # Missing for now: + # libfftw3-mpi-dev + # libopenmpi-dev + # libhdf5-mpi-dev + + # apt-get -y install \ + # $(python3 ./firedrake-repo/scripts/firedrake-configure --arch ${{ matrix.arch }} --show-system-packages) + # run a list of app packages and grep openmpi and fail - name: Install PETSc run: | git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git cd petsc # TODO update configure file - ./configure --with-make-np=8 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-mpich --download-slepc + ./configure --with-make-np=8 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi --download-slepc make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all - # TODO: This fails for some reason - # make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" + make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" - name: Install Firedrake id: install @@ -84,7 +99,6 @@ jobs: export CC=$PETSC_DIR/$PETSC_ARCH/bin/mpicc export CXX=$PETSC_DIR/$PETSC_ARCH/bin/mpicxx export MPICC=$CC - export MPI_HOME=$PETSC_DIR/$PETSC_ARCH export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_DIR=$PETSC_DIR/$PETSC_ARCH From 6ff1e91e0da37bf7e43fe875bd360c3e6dc1ab86 Mon Sep 17 00:00:00 2001 From: Olender Date: Wed, 23 Apr 2025 15:02:19 +0100 Subject: [PATCH 40/44] wip --- .github/workflows/build_cuda.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index e91cbf206c..108727e775 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -86,6 +86,17 @@ jobs: # TODO update configure file ./configure --with-make-np=8 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi --download-slepc make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all + export PETSC_DIR=/__w/firedrake/firedrake/petsc + export PETSC_ARCH=arch-firedrake-default + export MPI_HOME=$PETSC_DIR/$PETSC_ARCH + export CC=$PETSC_DIR/$PETSC_ARCH/bin/mpicc + export CXX=$PETSC_DIR/$PETSC_ARCH/bin/mpicxx + export MPICC=$CC + export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH + export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH + export HDF5_DIR=$PETSC_DIR/$PETSC_ARCH + export HYPRE_DIR=$PETSC_DIR/$PETSC_ARCH + export HDF5_MPI=ON make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" - name: Install Firedrake From 86ae6a8ba2eca3da58f4c3afbd34ca0122ec9704 Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 24 Apr 2025 14:30:14 +0100 Subject: [PATCH 41/44] wip --- .github/workflows/build_cuda.yml | 4 +--- tests/firedrake/conftest.py | 12 ++++++++++++ tests/firedrake/demos/test_demos_run.py | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 108727e775..342d936a32 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -84,7 +84,7 @@ jobs: git clone --depth 1 --branch $(python3 ./firedrake-repo/scripts/firedrake-configure --show-petsc-version) https://gitlab.com/petsc/petsc.git cd petsc # TODO update configure file - ./configure --with-make-np=8 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --download-hypre --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi --download-slepc + ./configure --with-make-np=8 --with-c2html=0 --with-debugging=0 --with-fortran-bindings=0 --with-shared-libraries=1 --with-strict-petscerrorcode PETSC_ARCH=arch-firedrake-default --COPTFLAGS=-O3 -march=native -mtune=native --CXXOPTFLAGS=-O3 -march=native -mtune=native --FOPTFLAGS=-O3 -march=native -mtune=native --download-bison --download-fftw --download-hdf5 --download-hwloc --download-metis --download-mumps --download-netcdf --download-pnetcdf --download-ptscotch --download-scalapack --download-suitesparse --download-superlu_dist --download-zlib --with-cuda --with-cuda-dir=/usr/local/cuda CUDAPPFLAGS=-Wno-deprecated-gpu-targets --download-openmpi --download-slepc make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default all export PETSC_DIR=/__w/firedrake/firedrake/petsc export PETSC_ARCH=arch-firedrake-default @@ -95,7 +95,6 @@ jobs: export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_DIR=$PETSC_DIR/$PETSC_ARCH - export HYPRE_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_MPI=ON make PETSC_DIR=/__w/firedrake/firedrake/petsc PETSC_ARCH=arch-firedrake-default check MPIEXEC="mpiexec --allow-run-as-root" @@ -113,7 +112,6 @@ jobs: export PATH=$PETSC_DIR/$PETSC_ARCH/bin:$PATH export SLEPC_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_DIR=$PETSC_DIR/$PETSC_ARCH - export HYPRE_DIR=$PETSC_DIR/$PETSC_ARCH export HDF5_MPI=ON python3 -m venv venv . venv/bin/activate diff --git a/tests/firedrake/conftest.py b/tests/firedrake/conftest.py index b056d96b94..ab6252642a 100644 --- a/tests/firedrake/conftest.py +++ b/tests/firedrake/conftest.py @@ -149,6 +149,10 @@ def pytest_configure(config): "markers", "skipcuda: mark as skipped if CUDA is not available" ) + config.addinivalue_line( + "markers", + "skipgpu: mark as skipped if a GPU enabled PETSC was installed" + ) def pytest_collection_modifyitems(session, config, items): @@ -156,9 +160,13 @@ def pytest_collection_modifyitems(session, config, items): try: get_petsc_variables()["CUDA_VERSION"] + # They look like the same thing (but opposite) for now, but they are not. + # This will skip some nongpurelated tests (hypre) if a gpu-aware petsc was installed. cuda_unavailable = False + gpu_based_petsc = True except: cuda_unavailable = True + gpu_based_petsc = False for item in items: if complex_mode: @@ -173,6 +181,10 @@ def pytest_collection_modifyitems(session, config, items): if cuda_unavailable: if item.get_closest_marker("skipcuda") is not None: item.add_marker(pytest.mark.skip(reason="CUDA not available")) + + if gpu_based_petsc: + if item.get_closest_marker("skipgpu") is not None: + item.add_marker(pytest.mark.skip(reason="Test skipped on gpu-based install")) for dep, marker, reason in dependency_skip_markers_and_reasons: if _skip_test_dependency(dep) and item.get_closest_marker(marker) is not None: diff --git a/tests/firedrake/demos/test_demos_run.py b/tests/firedrake/demos/test_demos_run.py index fe5665a0fe..8fc09fcd5f 100644 --- a/tests/firedrake/demos/test_demos_run.py +++ b/tests/firedrake/demos/test_demos_run.py @@ -117,6 +117,7 @@ def _exec_file(py_file): @pytest.mark.skipcomplex +@pytest.mark.skipgpu @pytest.mark.parametrize("demo", SERIAL_DEMOS, ids=["/".join(d.loc) for d in SERIAL_DEMOS]) def test_serial_demo(demo, env, monkeypatch, tmpdir, skip_dependency): _maybe_skip_demo(demo, skip_dependency) @@ -130,6 +131,7 @@ def test_serial_demo(demo, env, monkeypatch, tmpdir, skip_dependency): @pytest.mark.parallel(2) @pytest.mark.skipcomplex +@pytest.mark.skipgpu @pytest.mark.parametrize("demo", PARALLEL_DEMOS, ids=["/".join(d.loc) for d in PARALLEL_DEMOS]) def test_parallel_demo(demo, env, monkeypatch, tmpdir, skip_dependency): _maybe_skip_demo(demo, skip_dependency) From c96c15b1a54f710ba6c7b83ef57f0b1d8fa9972d Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 24 Apr 2025 14:30:23 +0100 Subject: [PATCH 42/44] wip --- tests/firedrake/multigrid/test_nested_split.py | 1 + tests/firedrake/regression/test_hypre_ads.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/tests/firedrake/multigrid/test_nested_split.py b/tests/firedrake/multigrid/test_nested_split.py index 77083dc447..6b65780a20 100644 --- a/tests/firedrake/multigrid/test_nested_split.py +++ b/tests/firedrake/multigrid/test_nested_split.py @@ -56,6 +56,7 @@ "fieldsplit_1_sub_pc_type": "ilu"}]) @pytest.mark.skipcomplex @pytest.mark.skipcomplexnoslate +@pytest.mark.skipgpu def test_nested_split_multigrid(parameters): mesh = UnitSquareMesh(10, 10) diff --git a/tests/firedrake/regression/test_hypre_ads.py b/tests/firedrake/regression/test_hypre_ads.py index 7468eed722..74ecbcd8fb 100644 --- a/tests/firedrake/regression/test_hypre_ads.py +++ b/tests/firedrake/regression/test_hypre_ads.py @@ -4,6 +4,7 @@ @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_linear(): mesh = UnitCubeMesh(10, 10, 10) V = FunctionSpace(mesh, "RT", 1) @@ -32,6 +33,7 @@ def test_homogeneous_field_linear(): @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_matfree(): mesh = UnitCubeMesh(10, 10, 10) V = FunctionSpace(mesh, "RT", 1) @@ -63,6 +65,7 @@ def test_homogeneous_field_matfree(): @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_nonlinear(): mesh = UnitCubeMesh(10, 10, 10) V = FunctionSpace(mesh, "RT", 1) @@ -89,6 +92,7 @@ def test_homogeneous_field_nonlinear(): @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_linear_convergence(): mesh = UnitCubeMesh(10, 10, 10) V = FunctionSpace(mesh, "RT", 1) From 61f65b1e6a6b4b0a80236f87173bef88c73f90a1 Mon Sep 17 00:00:00 2001 From: Olender Date: Thu, 24 Apr 2025 15:00:27 +0100 Subject: [PATCH 43/44] wip --- tests/firedrake/regression/test_hypre_ams.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/firedrake/regression/test_hypre_ams.py b/tests/firedrake/regression/test_hypre_ams.py index 549a1d7e8b..03d97c8596 100644 --- a/tests/firedrake/regression/test_hypre_ams.py +++ b/tests/firedrake/regression/test_hypre_ams.py @@ -5,6 +5,7 @@ @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_linear(): mesh = UnitCubeMesh(5, 5, 5) V = FunctionSpace(mesh, "N1curl", 1) @@ -39,6 +40,7 @@ def test_homogeneous_field_linear(): @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_matfree(): mesh = UnitCubeMesh(5, 5, 5) V = FunctionSpace(mesh, "N1curl", 1) @@ -76,6 +78,7 @@ def test_homogeneous_field_matfree(): @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_nonlinear(): mesh = UnitCubeMesh(5, 5, 5) V = FunctionSpace(mesh, "N1curl", 1) @@ -109,6 +112,7 @@ def test_homogeneous_field_nonlinear(): @pytest.mark.skiphypre @pytest.mark.skipcomplex +@pytest.mark.skipgpu def test_homogeneous_field_linear_convergence(): N = 4 mesh = UnitCubeMesh(2**N, 2**N, 2**N) From c57acf091ef4d08ca8fad723e9d8109e889008a6 Mon Sep 17 00:00:00 2001 From: Olender Date: Tue, 6 May 2025 15:18:41 -0300 Subject: [PATCH 44/44] jsut to debug something --- .github/workflows/build_cuda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_cuda.yml b/.github/workflows/build_cuda.yml index 342d936a32..fdc5690d89 100644 --- a/.github/workflows/build_cuda.yml +++ b/.github/workflows/build_cuda.yml @@ -36,7 +36,7 @@ jobs: FIREDRAKE_CI: 1 PYOP2_CI_TESTS: 1 PYOP2_SPMD_STRICT: 1 - EXTRA_PYTEST_ARGS: --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 firedrake-repo/tests/firedrake + EXTRA_PYTEST_ARGS: --use_gpu_aware_mpi=0 --splitting-algorithm least_duration --timeout=600 --timeout-method=thread -o faulthandler_timeout=660 firedrake-repo/tests/firedrake steps: - name: Fix HOME # For unknown reasons GitHub actions overwrite HOME to /github/home