Try using an editor descriptor, you can OverrideDefault with your own widget or look at other options available to remove a specific input

    [EditorDescriptorRegistration(
    TargetType = typeof(string),
    UIHint = "HyperLink",
    EditorDescriptorBehavior = EditorDescriptorBehavior.OverrideDefault)
]
    internal class CustomLinkEditorDescriptor : EditorDescriptor
    {
        public override void ModifyMetadata(ExtendedMetadata metadata, IEnumerable<Attribute> attributes)
        {
            base.ModifyMetadata(metadata, attributes);

            if (!metadata.EditorConfiguration.ContainsKey(("providers")))
            {
                return;
            }

            var providers = (metadata.EditorConfiguration["providers"] as IEnumerable<object>).ToList();

            metadata.EditorConfiguration["providers"] = providers;
        }
    }

This gave me the following view 

Thanks Minesh, but this doesn't help in hiding/removing only media