You can only add only few custom actions configuration with modern experience
EditControlBlock - Yes, these entries show up as custom menu items.
CommandUI.Ribbon - Yes, these entries show up as toolbar items.
Everthing else - Not supported.
But as alternative, If you are looking for something link below. refer to sample provided in pnp sample
<pnp:ProvisioningTemplate ID="CommandUIRibbonSamples" Version="1" xmlns:pnp="http://schemas.dev.office.com/PnP/2015/12/ProvisioningSchema">
<pnp:CustomActions>
<pnp:SiteCustomActions>
<pnp:CustomAction Name="CA_4" Description="ca 4" Location="CommandUI.Ribbon" RegistrationType="List" RegistrationId="101" Title="CA 4 Title" Sequence="6000" Enabled="true">
<pnp:CommandUIExtension>
<CommandUIDefinitions>
<CommandUIDefinition Location="Ribbon.Documents.Copies.Controls._children">
<Button
Id="Ribbon.Documents.Copies.OfficeDevPnPDownloadAll"
Command="OfficeDevPnP.Cmd.DownloadAll"
Image16by16="/_layouts/15/images/sharepointfoundation16.png"
LabelText="Download All"
Description="Download all files separately"
ToolTipTitle="Download All"
ToolTipDescription="Download all files separately"
TemplateAlias="o1"
Sequence="15"/>
</CommandUIDefinition>
</CommandUIDefinitions>
<CommandUIHandlers>
<CommandUIHandler
Command="OfficeDevPnP.Cmd.DownloadAll"
CommandAction="https://contoso.azurewebsites.net/pages/index.aspx" />
</CommandUIHandlers>
</pnp:CommandUIExtension>
</pnp:CustomAction>
<pnp:CustomAction Name="CA_6" Description="ca 6" Location="CommandUI.Ribbon" RegistrationType="ContentType" RegistrationId="0x0101" Title="CA 6 Title" Sequence="5000" Enabled="true">
<pnp:CommandUIExtension>
<CommandUIDefinitions>
<CommandUIDefinition Location="Ribbon.Tabs._children">
<Tab Id="Custom Tab" Title="Custom Tab" Description="Custom Tab">
<Scaling Id="Custom Tab.Scaling">
<MaxSize Id="Custom Group.Scaling.MaxSize" GroupId="Custom Group" Size="TwoLarge" />
<MaxSize Id="Custom Group 2.Scaling.MaxSize" GroupId="Custom Group 2" Size="OneLarge" />
<Scale Id="Custom Group.Scaling.Scale" GroupId="Custom Group" Size="TwoLarge" />
<Scale Id="Custom Group 2.Scaling.Scale" GroupId="Custom Group 2" Size="OneLarge" />
</Scaling>
<Groups Id="Custom Tab.Groups">
<Group Id="Custom Group 2" Title="Custom Group 2" Description="Custom Group 2" Sequence="7888" Template="Ribbon.Templates.OneLarge">
<Controls Id="Custom Group 2.Controls">
<Button Id="CustomButton3" LabelText="Custom Button 3" Image16by16="/_layouts/15/images/attach16.png" Image32by32="/_layouts/15/images/attach16.png" ToolTipTitle="Custom Button 3" ToolTipDescription="Custom Button 3" Command="CustomButton3.Command" TemplateAlias="c3" />
</Controls>
</Group>
<Group Id="Custom Group" Title="Custom Group 1" Description="Custom Group 1" Sequence="10000" Template="Ribbon.Templates.TwoLarge">
<Controls Id="Custom Group 1.Controls">
<Button Id="CustomButton1" LabelText="Custom Button 1" Image16by16="/_layouts/15/images/itslidelibrary.png" Image32by32="/_layouts/15/images/itslidelibrary.png" ToolTipTitle="Custom Button 1" ToolTipDescription="Custom Button 1" Command="CustomButton1.Command" TemplateAlias="c1" />
<Button Id="CustomButton2" LabelText="Custom Button 2" Image16by16="/_layouts/15/images/dldsln16.png" Image32by32="/_layouts/15/images/dldsln16.png" ToolTipTitle="Custom Button 2" ToolTipDescription="Custom Button 2" Command="CustomButton2.Command" TemplateAlias="c2" />
</Controls>
</Group>
</Groups>
</Tab>
</CommandUIDefinition>
<CommandUIDefinition Location="Ribbon.Templates._children">
<GroupTemplate Id="Ribbon.Templates.TwoLarge">
<Layout Title="TwoLarge" LayoutTitle="TwoLarge">
<Section Alignment="Top" Type="OneRow">
<Row>
<ControlRef DisplayMode="Large" TemplateAlias="c1" />
<ControlRef DisplayMode="Large" TemplateAlias="c2" />
</Row>
</Section>
</Layout>
</GroupTemplate>
</CommandUIDefinition>
<CommandUIDefinition Location="Ribbon.Templates._children">
<GroupTemplate Id="Ribbon.Templates.OneLarge">
<Layout Title="OneLarge" LayoutTitle="OneLarge">
<Section Alignment="Top" Type="OneRow">
<Row>
<ControlRef DisplayMode="Large" TemplateAlias="c3" />
</Row>
</Section>
</Layout>
</GroupTemplate>
</CommandUIDefinition>
</CommandUIDefinitions>
<CommandUIHandlers>
<CommandUIHandler Command="CustomButton1.Command" CommandAction="https://contoso.azurewebsites.net/pages/index.aspx" />
<CommandUIHandler Command="CustomButton2.Command" CommandAction="http://www.bing.com" />
<CommandUIHandler Command="CustomButton3.Command" CommandAction="http://dev.office.com/sharepoint" />
</CommandUIHandlers>
</pnp:CommandUIExtension>
</pnp:CustomAction>
</pnp:SiteCustomActions>
</pnp:CustomActions>
</pnp:ProvisioningTemplate>