Changes for latest FEniCSx v0.7.x

compilers.yaml

@@ -1,9 +1,23 @@
-gcc-10:
+gcc-13:
   version:
-    - 10
+    - 13
   cpp:
-    - g++-10
+    - /home/mhabera/spack/opt/spack/linux-amzn2-neoverse_v1/gcc-7.3.1/gcc-13.2.0-nxk6zirma34efpxq7yod2wuyjhwwagwh/bin/g++
   cc:
-    - gcc-10
+    - /home/mhabera/spack/opt/spack/linux-amzn2-neoverse_v1/gcc-7.3.1/gcc-13.2.0-nxk6zirma34efpxq7yod2wuyjhwwagwh/bin/gcc
   flags:
-    - -Ofast -march=native -mprefer-vector-width=256
+    - -Ofast -mcpu=neoverse-v1
+    - -Ofast -mcpu=neoverse-v1 -fno-tree-vectorize
+    - -O2 -fno-tree-vectorize
+
+llvm-17:
+  version:
+    - 17
+  cpp:
+    - /home/mhabera/spack/opt/spack/linux-amzn2-neoverse_v1/gcc-13.2.0/llvm-17.0.4-ti5dvewumkoer6wsn6v4eipjrrkb4cg5/bin/clang++
+  cc:
+    - /home/mhabera/spack/opt/spack/linux-amzn2-neoverse_v1/gcc-13.2.0/llvm-17.0.4-ti5dvewumkoer6wsn6v4eipjrrkb4cg5/bin/clang
+  flags:
+    - -Ofast -mcpu=neoverse-v1 -Wl,-rpath,/home/mhabera/spack/opt/spack/linux-amzn2-neoverse_v1/gcc-7.3.1/gcc-13.2.0-nxk6zirma34efpxq7yod2wuyjhwwagwh/lib64
+    - -Ofast -mcpu=neoverse-v1 -fno-vectorize -fno-slp-vectorize -Wl,-rpath,/home/mhabera/spack/opt/spack/linux-amzn2-neoverse_v1/gcc-7.3.1/gcc-13.2.0-nxk6zirma34efpxq7yod2wuyjhwwagwh/lib64
+    - -O2 -fno-vectorize -fno-slp-vectorize -Wl,-rpath,/home/mhabera/spack/opt/spack/linux-amzn2-neoverse_v1/gcc-7.3.1/gcc-13.2.0-nxk6zirma34efpxq7yod2wuyjhwwagwh/lib64

ffcx/compile.py

@@ -2,9 +2,9 @@
 from ffcx.codegeneration.backend import FFCXBackend
 from ffcx.analysis import analyze_ufl_objects
 from ffcx.ir.representation import compute_ir
-from ffcx.codegeneration.integrals import IntegralGenerator
-from ffcx.element_interface import create_element
-from ffcx.codegeneration.C.format_lines import format_indented_lines
+from ffcx.codegeneration.integral_generator import IntegralGenerator
+from ffcx.codegeneration.C.c_implementation import CFormatter
+from basix.ufl import convert_ufl_element
 from ffcx.options import get_options
 import basix
 import ufl
@@ -80,13 +80,7 @@
 
 
 def compute_integral_body(ir, backend):
-    # Configure kernel generator
-    ig = IntegralGenerator(ir, backend)
-    # Generate code ast for the tabulate_tensor body
-    parts = ig.generate()
-    # Format code as string
-    body = format_indented_lines(parts.cs_format(ir.precision), 1)
-    return body
+    return parts
 
 
 def compile_form(form: ufl.Form, name: str,
@@ -118,11 +112,15 @@ def compile_form(form: ufl.Form, name: str,
         geom_type += str(batch_size)
         scalar_type += str(batch_size)
 
-    settings = {"scalar_type": scalar_type, "geom_type": geom_type}
-    arguments = _arguments.format(**settings)
+    arguments = _arguments.format(scalar_type=scalar_type, geom_type=geom_type)
     signature = "inline void " + name + arguments
-    body = compute_integral_body(integral_ir, backend)
-    code = signature + " {\n" + body + "\n}\n"
+    # Configure kernel generator
+    ig = IntegralGenerator(integral_ir, backend)
+    # Generate code ast for the tabulate_tensor body
+    parts = ig.generate()
+    formatter = CFormatter(scalar_type)
+    body_c = formatter.c_format(parts)
+    code = signature + " {\n" + body_c + "\n}\n"
 
     return code
 
@@ -146,7 +144,7 @@ def generate_code(action, scalar_type, global_size, batch_size):
             [problem.a], parameters).form_data[0].num_coefficients
         rank = 2
 
-    element = create_element(problem.element)
+    element = convert_ufl_element(problem.element)
     num_nodes = element.cell().num_vertices()
     geom_type = scalar_type.replace(' _Complex', '')
 

ffcx/main.cpp

@@ -61,4 +61,4 @@ int main(int argc, char *argv[])
   MPI_Finalize();
 
   return 0;
-}
+}

forms/Elasticity.ufl

@@ -1,4 +1,6 @@
-from ufl import *
+import basix.ufl
+from ufl import (Mesh, FiniteElement, FunctionSpace, TrialFunction, TestFunction, Coefficient,
+                sym, grad, inner, dx, action, tetrahedron, VectorElement)
 
 element = VectorElement("Lagrange", $cell, $degree)
 mesh = Mesh(VectorElement("Lagrange", $cell, 1))
@@ -17,4 +19,4 @@ def eps(v):
 a = k*inner(eps(u), eps(v))*dx
 
 un = Coefficient(V)
-L = action(a, un)
+L = action(a, un)

run.py

@@ -20,6 +20,9 @@
 
     parser.add_argument('--conf', dest='conf', type=str, default="compilers.yaml",
                         help="Configuration file describing the compilers and flags.")
+
+    parser.add_argument('--output_file', dest='output_file', type=str, default="output/output.csv",
+                        help="Configuration file describing the compilers and flags.")
 
     parser.add_argument('--degree', dest='degree', default=range(1, 4), nargs='+',
                         help='Polynomial degree to evaluate the operators.')
@@ -55,9 +58,10 @@
     scalar_type = args.scalar_type
     mpi_size = args.mpi_size
     cell_type = args.cell_type
+    output_file = args.output_file
 
     machine = utils.machine_name()
-    out_file = utils.create_ouput(problem)
+    out_file = utils.create_output(problem, output_file)
     compilers = utils.parse_compiler_configuration(conf_file)
 
     # Set rank to 1 for matrix free, 2 otherwise

run.sh

@@ -0,0 +1,14 @@
+#!/bin/bash -l
+#SBATCH --job-name=mpi_job_test      # Job name
+#SBATCH --cpus-per-task=1            # Number of cores per MPI task 
+#SBATCH --nodes=1                    # Maximum number of nodes to be allocated
+#SBATCH --ntasks-per-node=8          # Maximum number of tasks on each node
+#SBATCH --output=mpi_test_%j.log     # Path to the standard output and error files relative to the working directory
+#SBATCH -p small
+
+spack load cmake
+spack load gcc@13.2.0+binutils
+spack load llvm
+spack env activate ffcx
+python3 run.py --nrepeats 1 --problem Elasticity  --degree 2 --form_compiler=ffcx --action --global_size 1000000 --output_file=output/elasticity_2_action.csv
+python3 run.py --nrepeats 1 --problem Elasticity  --degree 6 --form_compiler=ffcx --action --global_size 10000000 --output_file=output/elasticity_6_action.csv

utils.py

@@ -41,14 +41,10 @@ def machine_name():
     return machine
 
 
-def create_ouput(problem):
+def create_output(problem, out_file):
     header = "machine,problem,compiler,version,flags,degree,fcomp,scalar,batch_size,rank,cell_type,ncells,time"
-    path = "output/"
-    out_file = path + str(problem) + ".txt"
 
     if not os.path.exists(out_file):
-        if not os.path.isdir(path):
-            os.mkdir(path)
         with open(out_file, "a") as f:
             f.write(header)
     return out_file